@vertesia/workflow 0.24.0-dev.202601221707

package/src/dsl/vars.test.ts

@@ -0,0 +1,245 @@
+import { describe, expect, test } from "vitest";
+import { Vars } from "./vars.ts";
+
+describe('Workflow vars', () => {
+
+    test('access initial variables', () => {
+        const vars = new Vars({
+            objectId: "123",
+            timestamp: 123456,
+        });
+        expect(vars.getValue("objectId")).toBe("123");
+        expect(vars.getValue("timestamp")).toBe(123456);
+        expect(vars.getValue("foo")).toBeUndefined();
+    })
+
+    test('set and access variables', () => {
+        const vars = new Vars({
+            objectId: "123",
+            timestamp: 123456,
+        });
+        expect(vars.getValue("objectId")).toBe("123");
+        vars.setValue("objectId", "456");
+        expect(vars.getValue("objectId")).toBe("456");
+    })
+
+    test('access and modify ref variables', () => {
+        const vars = new Vars({
+            objectId: "123",
+            config: {
+                name: "foo",
+            },
+            objectIdAlias: "${objectId}",
+            configNameAlias: "${config.name}",
+        });
+        expect(vars.getValue("objectIdAlias")).toBe("123");
+        expect(vars.getValue("configNameAlias")).toBe("foo");
+        vars.setValue("objectId", "456");
+        vars.getValue("config").name = "bar";
+        expect(vars.getValue("objectIdAlias")).toBe("456");
+        expect(vars.getValue("configNameAlias")).toBe("bar");
+    })
+
+    test('replace literal with ref', () => {
+        const vars = new Vars({
+            objectId: "123",
+            otherId: "1234",
+        });
+        expect(vars.getValue("objectId")).toBe("123");
+        expect(vars.getValue("otherId")).toBe("1234");
+        vars.setValue("objectId", "456");
+        vars.load({ "otherId": "${objectId}" });
+        expect(vars.getValue("objectId")).toBe("456");
+        expect(vars.getValue("otherId")).toBe("456");
+    });
+
+    test("undefined ref variable", () => {
+        const vars = new Vars({
+            otherId: "${objectId}",
+        });
+        expect(vars.getValue("otherId")).toBeUndefined();
+    })
+
+    test("default values", () => {
+        const vars = new Vars({
+            squote: "${objectId ?? '123'}",
+            dquote: "${objectId ?? \"123\"}",
+            number: "${objectId ?? 123}",
+            boolean: "${objectId ?? true}",
+            array: "${objectId ?? [1,2]}",
+        });
+        // object value and path expression are not supported
+        expect(vars.getValue("squote")).toBe("123");
+        expect(vars.getValue("dquote")).toBe("123");
+        expect(vars.getValue("number")).toBe(123);
+        expect(vars.getValue("boolean")).toBe(true);
+        expect(vars.getValue("array")).toEqual([1, 2]);
+    })
+
+    test("array map", () => {
+        const vars = new Vars({
+            docs: [{ text: "hello" }, { text: "world" }],
+        });
+        // object valus and path expression are not supported
+        expect(vars.getValue("docs.text").join(' ')).toBe("hello world");
+    })
+
+    test("resolveParams", () => {
+        const vars = new Vars({
+            objectIds: ["123", "456"],
+            objectId: "123",
+            data: { message: "hello" },
+        });
+        const params = {
+            query: {
+                _id: "${objectId}",
+                data: "${data}",
+            },
+            secondDocId: "${objectIds.1}",
+            secondDocIdByBracket: "${objectIds[1]}",
+        }
+        const resolved = vars.resolveParams(params);
+        expect(Object.keys(resolved).length).toBe(3);
+        expect(resolved.query).toBeTypeOf("object");
+        expect(resolved.query._id).toBe("123");
+        expect(resolved.query.data).toBeTypeOf("object");
+        expect(resolved.query.data.message).toBe("hello");
+        expect(resolved.secondDocId).toBe("456");
+        expect(resolved.secondDocIdByBracket).toBe("456");
+
+    });
+
+    test("resolve", () => {
+        const vars = new Vars({
+            objectId: "123",
+            message: "hello",
+            prompt_data: { message: "${message}" },
+            data2: "${prompt_data}",
+            unknown: "${notdefined}"
+        });
+        const resolved = vars.resolve();
+        console.log('@@@@@@@@@@@@@@@@@@@', resolved);
+        expect(Object.keys(resolved).length).toBe(4); // since unknown will be undefined it will not be included
+        expect(resolved.objectId).toBe("123");
+        expect(resolved.prompt_data).toBeTypeOf("object");
+        expect(resolved.prompt_data.message).toBe("hello");
+        expect(resolved.data2).toBeTypeOf("object");
+        expect(resolved.data2.message).toBe("hello");
+        expect(resolved.unknown).toBeUndefined();
+    });
+
+
+    test("createImportVars", () => {
+        const vars = new Vars({
+            person: { name: "John", age: 30 },
+            objectId: "123",
+            data: { message: "hello" },
+            data2: "${data}",
+        });
+        const params = vars.createImportVars(["data2", { "name": "person.name", "age": "person.age" }, { message: "data.message" }]);
+        expect(Object.keys(params).length).toBe(4);
+        expect(params.name).toBe("John");
+        expect(params.age).toBe(30);
+        expect(params.message).toBe("hello");
+        expect(params.data2).toStrictEqual({ message: "hello" });
+    });
+
+    test("evaluate null and exists conditions", () => {
+        const vars = new Vars({
+            objectId: "123",
+            docType: {
+                id: "bla",
+                idNull: null,
+            }
+        });
+        expect(vars.match({ objectId: { $exists: true } })).toBe(true);
+        expect(vars.match({ objectId: { $exists: false } })).toBe(false);
+        expect(vars.match({ objectIds: { $exists: true } })).toBe(false);
+        expect(vars.match({ objectIds: { $exists: false } })).toBe(true);
+        expect(vars.match({ "docType.id": { $exists: true } })).toBe(true);
+        expect(vars.match({ "docType.idNull": { $exists: true } })).toBe(true);
+
+        expect(vars.match({ objectId: { $null: true } })).toBe(false);
+        expect(vars.match({ objectId: { $null: false } })).toBe(true);
+        expect(vars.match({ objectIds: { $null: false } })).toBe(false);
+        expect(vars.match({ objectIds: { $null: true } })).toBe(true);
+        expect(vars.match({ "docType.id": { $null: true } })).toBe(false);
+        expect(vars.match({ "docType.idNull": { $null: true } })).toBe(true);
+    });
+
+    test("evaluate $eq conditions", () => {
+        const vars = new Vars({
+            objectId: "123",
+            otherId: "${objectId}",
+            config: {
+                name: { foo: "bar" },
+                tags: ["a", "b", "c"]
+            }
+        });
+        expect(vars.match({ objectId: { $eq: "123" } })).toBe(true);
+        expect(vars.match({ objectId: { $eq: "456" } })).toBe(false);
+        expect(vars.match({ objectId: { $ne: "123" } })).toBe(false);
+        expect(vars.match({ objectId: { $ne: "456" } })).toBe(true);
+
+        expect(vars.match({ otherId: { $eq: "123" } })).toBe(true);
+        expect(vars.match({ otherId: { $eq: "456" } })).toBe(false);
+        expect(vars.match({ otherId: { $ne: "123" } })).toBe(false);
+        expect(vars.match({ otherId: { $ne: "456" } })).toBe(true);
+
+        expect(vars.match({ "config": { $eq: { name: { foo: "bar" }, tags: ["a", "b", "c"] } } })).toBe(true);
+    });
+
+    test("evaluate $regexp conditions", () => {
+        const vars = new Vars({
+            expr: "a=2",
+        });
+        expect(vars.match({ expr: { $regexp: "^.*\\s*=\\s*.*$" } })).toBe(true);
+    });
+
+    test("evaluate string compare conditions", () => {
+        const vars = new Vars({
+            expr: "<tag />",
+        });
+        expect(vars.match({ expr: { $startsWith: "<" } })).toBe(true);
+        expect(vars.match({ expr: { $contains: "tag" } })).toBe(true);
+        expect(vars.match({ expr: { $endsWith: "/>" } })).toBe(true);
+    });
+
+    test("evaluate $in conditions", () => {
+        const vars = new Vars({
+            name: "foo",
+        });
+        expect(vars.match({ name: { $in: ["foo", "bar"] } })).toBe(true);
+        expect(vars.match({ name: { $nin: ["hello", "world"] } })).toBe(true);
+    });
+
+    test("evaluate $lt, $gt conditions", () => {
+        const vars = new Vars({
+            value: 5,
+        });
+        expect(vars.match({ value: { $lt: 6 } })).toBe(true);
+        expect(vars.match({ value: { $gt: 4 } })).toBe(true);
+        expect(vars.match({ value: { $lte: 5 } })).toBe(true);
+        expect(vars.match({ value: { $gte: 5 } })).toBe(true);
+    });
+
+    test("evaluate $or conditions", () => {
+        const vars = new Vars({
+            value: 5,
+            name: "foo"
+        });
+        expect(vars.match({ value: { $or: [{ $lt: 4 }, { $gt: 6 }] } })).toBe(false);
+        expect(vars.match({ name: { $or: [{ $eq: "bar" }, { $eq: "foo" }] } })).toBe(true);
+        expect(vars.match({ name: { $or: [{ $eq: "bar" }, { $eq: "fooo" }] } })).toBe(false);
+    });
+
+    test("evaluate nested conditions", () => {
+        const vars = new Vars({
+            config: {
+                name: "foo",
+            },
+        });
+        expect(vars.match({ "config.name": { $eq: "foo" } })).toBe(true);
+    });
+
+});

package/src/dsl/vars.ts

@@ -0,0 +1,340 @@
+import { ImportSpec } from "@vertesia/common";
+import { matchCondition } from "./conditions.js";
+import { ObjectKey, ObjectVisitor, ObjectWalker } from "./walk.js";
+
+const FALLBACK_VALUE_SEP = "??";
+
+function decodeLiteralValue(value: string) {
+    if (value.startsWith("'") && value.endsWith("'")) {
+        value = '"' + value.slice(1, -1).replace(/(?<!\\)"/g, '\\"') + '"'
+    }
+    return JSON.parse(value);
+}
+
+export function splitPath(path: string) {
+    if (path.indexOf('[') > -1) { // support array like paths
+        path = path.replace(/\[(\d+)\]/g, '.$1');
+    }
+    return path.split('.');
+}
+
+/**
+ * Get the property named by "name" of the given object
+ * If an array is indexed using a string key then a map is done and an array with the content of the properties with that name are returned
+ * Ex: docs.text => will return an array of text properties of the docs array
+ * @param object the object
+ * @param name the name of the property.
+ * @returns the property value
+ */
+function _prop(object: any, name: string) {
+    if (object === undefined) {
+        return undefined;
+    }
+    object = _valueOf(object); // resolve Value objects
+    if (Array.isArray(object)) {
+        const index = +name;
+        if (isNaN(index)) {
+            // map array to property
+            return object.map(item => item[name]);
+        } else {
+            return _valueOf(object[index]);
+        }
+    } else {
+        return _valueOf(object[name]);
+    }
+
+}
+
+function _valueOf(value: any) {
+    return value instanceof Value ? value.value : value;
+}
+
+export function resolveField(object: any, path: string[]) {
+    let p = object as any;
+    if (!p) return p;
+    if (!path.length) return _valueOf(p);
+    const last = path.length - 1;
+    for (let i = 0; i < last; i++) {
+        p = _prop(p, path[i])
+        if (!p) {
+            return undefined;
+        }
+    }
+    return _prop(p, path[last]);
+}
+
+abstract class Value<T = any> {
+    abstract value: T;
+    abstract stringify(): string;
+}
+class LiteralValue<T = any> extends Value<T> {
+    constructor(public value: T) {
+        super();
+    }
+    stringify() {
+        return String(this.value);
+    }
+}
+
+class RefValue extends Value {
+    constructor(public vars: Vars, public path: string[], public defaultValue?: any) {
+        super();
+    }
+    get value() {
+        const v = this.vars.getValueFromPath(this.path);
+        if (v === undefined) {
+            if (this.defaultValue !== undefined) {
+                return this.defaultValue;
+            } else {
+                return this.vars.preserveRefs ? this.stringify() : undefined;
+            }
+        } else {
+            return v;
+        }
+    }
+
+    stringify() {
+        return "${" + this.path.join('.') + "}";
+    }
+}
+
+class ExprValue extends Value {
+    constructor(public vars: Vars, public parts: Value[]) {
+        super();
+    }
+    get value() {
+        const out: string[] = [];
+        for (const seg of this.parts) {
+            const v = seg.value;
+            if (v !== undefined) {
+                out.push(String(seg.value));
+            }
+        }
+        return out.join('');
+    }
+
+    stringify() {
+        const out = [];
+        for (const seg of this.parts) {
+            out.push(seg.stringify());
+        }
+        return "${" + out.join('') + "}";
+    }
+
+}
+
+
+export class Vars {
+    map: Record<string, any>;
+    /**
+     * This property is used when resolving params. It contains the list of references that should not be resolved to their value
+     * but instead they need to return the string representation of the expression to be able to regenerate another Vars instance with the same expression
+     */
+    //TODO this feature is no more used - it was replaced by importVars so we can now delete `preserveRefs`
+    preserveRefs: Set<string> | undefined;
+
+    constructor(vars?: Record<string, any>) {
+        this.map = vars ? this.parse(vars) : {};
+    }
+
+    parse(vars: Record<string, any>): Record<string, any> {
+        return new ObjectWalker().map(vars, (_key, value) => {
+            if (typeof value === 'string') {
+                return this.createValue(this, value);
+            } else {
+                return value;
+            }
+        });
+    }
+
+    load(vars: Record<string, any>) {
+        const toAppend = this.parse(vars);
+        this.map = Object.assign(this.map, toAppend);
+        return this;
+    }
+
+    /**
+     * Set a literal value (cannot set a ref)
+     * To add refs use `append()`
+     * @param name
+     * @param value
+     */
+    setValue(name: string, value: any) {
+        this.map[name] = value;
+    }
+
+    getValue(path: string) {
+        return resolveField(this.map, splitPath(path));
+    }
+
+    getValueFromPath(path: string[]) {
+        return resolveField(this.map, path);
+    }
+
+    has(name: string) {
+        return this.map[name] !== undefined;
+    }
+
+    match(match: Record<string, any>) {
+        for (const name of Object.keys(match)) {
+            const value = this.getValue(name);
+            if (!matchCondition(value, match[name])) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    resolveParams(params: Record<string, any>, preserveRefs?: Set<string>) {
+        this.preserveRefs = preserveRefs;
+        try {
+            return new ObjectWalker().map(params, (_key, value) => {
+                if (typeof value === 'string') {
+                    const v = this.createValue(this, value)
+                    return v instanceof Value ? v.value : v;
+                } else {
+                    return value;
+                }
+            });
+        } finally {
+            this.preserveRefs = undefined;
+        }
+    }
+
+    resolve(preserveRefs?: Set<string>): Record<string, any> {
+        function map(_key: ObjectKey, value: any) {
+            if (value instanceof Value) {
+                const v = value.value;
+                if (v && typeof v === 'object') {
+                    if (Array.isArray(v) || v.constructor === Object) {
+                        // an array or plain object - recurse into the
+                        // value to find other nested Ref values if any...
+                        return new ObjectWalker().map(v, map);
+                    }
+                } else {
+                    return v;
+                }
+            } else {
+                return value
+            }
+        }
+        try {
+            this.preserveRefs = preserveRefs;
+            return new ObjectWalker().map(this.map, map);
+        } finally {
+            this.preserveRefs = undefined;
+        }
+    }
+
+    createRefValue(ref: string) {
+        const index = ref.indexOf(FALLBACK_VALUE_SEP);
+        let defaultValue: any;
+        if (index > -1) {
+            defaultValue = decodeLiteralValue(ref.substring(index + FALLBACK_VALUE_SEP.length).trim());
+            ref = ref.substring(0, index).trim();
+        }
+        if (ref === '.' || ref.indexOf('..') > -1) {
+            throw new Error("Invalid variable reference: " + ref)
+        }
+        return new RefValue(this, splitPath(ref), defaultValue);
+    }
+
+    createValue(vars: Vars, obj: any) {
+        if (!obj) {
+            return obj;
+        }
+        if (typeof obj === 'string') {
+            if (obj.indexOf('${') > -1) {
+                const segments = obj.split(/(\${[^}]+})/);
+                if (segments.length === 1) {
+                    const seg = segments[0];
+                    return this.createRefValue(seg.substring(2, seg.length - 1));
+                }
+                const parts: Value[] = [];
+                for (const seg of segments) {
+                    if (seg === '') continue;
+                    if (seg.startsWith('${') && seg.endsWith('}')) {
+                        parts.push(this.createRefValue(seg.substring(2, seg.length - 1)));
+                    } else {
+                        parts.push(new LiteralValue(seg));
+                    }
+                }
+                if (parts.length === 1) {
+                    return parts[0];
+                } else {
+                    return new ExprValue(vars, parts);
+                }
+            } else {
+                return obj;
+            }
+        } else {
+            return obj;
+        }
+    }
+
+    createImportVars(importSpec: ImportSpec | undefined) {
+        if (!importSpec || importSpec.length === 0) {
+            return {};
+        }
+        const result: Record<string, any> = {};
+
+        for (const importVar of importSpec) {
+            if (typeof importVar === "string") {
+                addImportVar(importVar, undefined, this, result);
+            } else {
+                for (const key of Object.keys(importVar)) {
+                    addImportVar(importVar[key], key, this, result);
+                }
+            }
+        }
+        return result;
+    }
+
+    getUnknownReferences(obj: any) {
+        const visitor = new UnknownReferencesVisitor(this);
+        new ObjectWalker().walk(obj, visitor);
+        return visitor.result;
+    }
+}
+
+function addImportVar(varPath: string, asName: string | undefined, vars: Vars, result: Record<string, any>) {
+    let isRequired = false;
+    if (varPath.endsWith("!")) {
+        isRequired = true;
+        varPath = varPath.slice(0, -1);
+    }
+    let value = vars.getValue(varPath);
+    if (value === undefined && isRequired) {
+        throw new Error(`Import variable ${varPath} is required but not found`);
+    }
+    result[asName || varPath] = value;
+}
+
+
+class UnknownReferencesVisitor implements ObjectVisitor {
+
+    result: { name: string, expression: string }[] = [];
+
+    constructor(public vars: Vars) {
+    }
+
+    onValue(_key: ObjectKey, value: any) {
+        const vars = this.vars;
+        if (typeof value === "string") {
+            const v = vars.createValue(vars, value);
+            if (v instanceof ExprValue) {
+                for (const p of v.parts) {
+                    if (p instanceof RefValue) {
+                        if (!vars.has(p.path[0])) {
+                            this.result.push({ name: p.path.join('.'), expression: p.stringify() });
+                        }
+                    }
+                }
+            } else if (v instanceof RefValue) {
+                if (!vars.has(v.path[0])) {
+                    this.result.push({ name: v.path.join('.'), expression: v.stringify() });
+                }
+            }
+        }
+    }
+}

package/src/dsl/walk.test.ts

@@ -0,0 +1,81 @@
+import { describe, expect, test } from "vitest";
+import { ObjectWalker } from "./walk.ts";
+
+describe('walk object', () => {
+
+    test('find string values', () => {
+        const obj = {
+            name: "foo",
+            age: 42,
+            children: [{
+                name: "bar",
+                age: 12,
+            },
+            {
+                name: "baz",
+                age: 15,
+            }],
+            folder: {
+                subfolder: {
+                    name: "file",
+                }
+            }
+        }
+        const values = ["foo", "bar", "baz", "file"].sort().join(',');
+        const found: string[] = [];
+        new ObjectWalker().walk(obj, {
+            onValue(key, value) {
+                if (typeof value === "string") {
+                    found.push(value);
+                }
+            }
+        })
+        expect(found.sort().join(',')).toBe(values);
+    })
+
+    test('map numbers to string values', () => {
+        const obj = {
+            name: "foo",
+            age: 42,
+            children: [{
+                name: "bar",
+                age: 12,
+            },
+            {
+                name: "baz",
+                age: 15,
+            }],
+            folder: {
+                subfolder: {
+                    name: 123,
+                }
+            }
+        }
+        const r = new ObjectWalker().map(obj, (_key, value) => {
+            if (typeof value === "number") {
+                return String(value)
+            }
+            return value;
+        });
+        expect(r.age).toBe("42");
+        expect(r.children[0].age).toBe("12");
+        expect(r.children[1].age).toBe("15");
+        expect(r.folder.subfolder.name).toBe("123");
+    })
+
+    test('map numbers in an array to string values', () => {
+        const obj = [123, { x: 1 }, { y: 2 }, { z: 3 }]
+        const r = new ObjectWalker().map(obj, (_key, value) => {
+            if (typeof value === "number") {
+                return String(value)
+            }
+            return value;
+        });
+        expect(r.length).toBe(4);
+        expect(r[0]).toBe("123");
+        expect(r[1].x).toBe("1");
+        expect(r[2].y).toBe("2");
+        expect(r[3].z).toBe("3");
+    })
+
+})