flow-frame-core 0.1.0

package/dist/services/configuration/executor.js

@@ -0,0 +1,795 @@
+import { clearAllText, get, mouseClick, pasteText, pause, pressReturn } from '../../utils.js';
+import path from 'path';
+import fs from 'fs';
+import { fileModalOperate, isWindows, moveMouseDesktop, pressEscapeAsync, scroll } from '../../operations.js';
+import robot from "robotjs";
+import { captureScreenshot, captureScreenshotBase64 } from '../../inference/capturescreenshot.js';
+import { classifyImageQuery, findImageInImageQuery, generalClassifyImageQuery } from '../runPrompt.js';
+import { KEYS } from '../../constants.js';
+import { similarityScore } from '../../operations/blender-ops.js';
+import { SmartError } from '../error.js';
+import { analyzeImageForTextAndIcons } from '../../promptRoutes.js';
+import { ingestScreenshot } from '../self-learning/injest.js';
+import { recordTransition } from '../self-learning/record_transition.js';
+import { recognizeScreen } from '../self-learning/recognize.js';
+/**
+ * Sets a value at a nested property path in an object
+ * @param {Object} obj - The object to set the value in
+ * @param {string} propertyPath - Dot-delimited path (e.g., "result.image")
+ * @param {*} value - The value to set
+ */
+function setNestedProperty(obj, propertyPath, value) {
+    if (!propertyPath)
+        return;
+    let keys = propertyPath.split('.');
+    let current = obj;
+    for (let i = 0; i < keys.length - 1; i++) {
+        if (!current[keys[i]]) {
+            current[keys[i]] = {};
+        }
+        current = current[keys[i]];
+    }
+    current[keys[keys.length - 1]] = value;
+    console.log(`Stored value in output at ${propertyPath}`);
+}
+/**
+ * Gets a value from a nested property path in an object
+ * @param {Object} obj - The object to get the value from
+ * @param {string} propertyPath - Dot-delimited path (e.g., "result.image")
+ * @returns {*} The value at the nested property path, or undefined if not found
+ */
+function getNestedProperty(obj, propertyPath) {
+    if (!propertyPath || !obj)
+        return undefined;
+    let keys = propertyPath.split('.');
+    let current = obj;
+    for (let i = 0; i < keys.length; i++) {
+        if (current === null || current === undefined || typeof current !== 'object') {
+            return undefined;
+        }
+        current = current[keys[i]];
+    }
+    return current;
+}
+/**
+ * Execute key commands, handling both single commands and multiple commands
+ * @param {string} keys - The key command(s) to execute (e.g., "{up}", "{up}{up}{up}{up}", or plain text)
+ * @param {Object} inputs - Input data object
+ * @param {Object} params - Step parameters
+ */
+async function executeKeyCommands(keys, inputs, params) {
+    if (!keys)
+        return;
+    // Check if this contains multiple key commands (e.g., "{up}{up}{up}{up}")
+    const keyCommandRegex = /^\{[^}]+\}(\{[^}]+\})*$/;
+    if (keyCommandRegex.test(keys)) {
+        // Parse multiple key commands
+        const commandMatches = keys.match(/\{[^}]+\}/g);
+        if (commandMatches) {
+            console.log(`Executing ${commandMatches.length} key commands: ${keys}`);
+            for (const command of commandMatches) {
+                console.log(`Executing single key command: ${command}`);
+                await executeSingleKeyCommand(command);
+                // Small delay between key presses
+                await pause(() => { }, params.delay || 50);
+            }
+        }
+    }
+    else {
+        // Check if it's a single key command
+        const singleCommandRegex = /^\{[^}]+\}$/;
+        if (singleCommandRegex.test(keys)) {
+            console.log(`Executing single key command: ${keys}`);
+            await executeSingleKeyCommand(keys);
+        }
+        else {
+            // Not a key command, treat as text to type
+            let textToType = (inputs?.inputs || inputs.a)[params.textToType] || params.text || '';
+            console.log(`Typing text: ${textToType}`);
+            robot.typeString(textToType);
+        }
+    }
+}
+/**
+ * Execute a single key command
+ * @param {string} command - Single key command (e.g., "{up}", "{enter}")
+ */
+async function executeSingleKeyCommand(command) {
+    switch (command) {
+        case '{enter}':
+            pressReturn();
+            break;
+        case '{tab}':
+            robot.keyTap('tab');
+            break;
+        case '{right}':
+            robot.keyTap('right');
+            break;
+        case '{left}':
+            robot.keyTap('left');
+            break;
+        case '{up}':
+            robot.keyTap('up');
+            break;
+        case '{down}':
+            robot.keyTap('down');
+            break;
+        default:
+            console.log(`Unknown key command: ${command}`);
+            break;
+    }
+}
+export async function executeConfiguration(workflowConfig, config, ctx, inputs, workflows) {
+    // Implementation for executing the workflow with the given config
+    const { workflow, jobs, subJobs } = workflowConfig;
+    const output = {};
+    console.log(`Executing workflow: ${workflow.id} with config:`, config);
+    const job = jobs?.find(job => job.id === config.version);
+    if (job) {
+        console.log(`Found job: ${job.id}`);
+    }
+    else {
+        console.warn(`Job not found for version: ${config.version}`);
+        // throw new Error(`Job not found for version: ${config.version}`);
+    }
+    // const subs = subJobs.filter(sub => sub.workflowId === workflow.id && sub.parentJobId === job.id);
+    let steps = workflow?.content?.steps || workflow?.steps;
+    let i = 0;
+    while (i < steps.length) {
+        const step = steps[i];
+        console.log(`Executing step: ${step.id}, step number ${i}`);
+        // Execute each step with the provided config
+        if (inputs?.inputs) {
+            let params = step?.params;
+            let tags = params?.selfLearningTags || [];
+            let hasTag = Object.keys(inputs?.inputs || {}).some(t => tags.includes(t));
+            if (!hasTag && !inputs?.inputs[params?.selfLearningMetadata?.name] &&
+                !inputs?.inputs[params?.selfLearningMetadata?.associated]) {
+                console.log(`No inputs found for step ${step.id}, skipping step.`);
+                console.log(`name: ${params?.selfLearningMetadata?.name}`);
+                console.log(`associated: ${params?.selfLearningMetadata?.associated}`);
+                i++;
+                continue;
+            }
+            console.log(`name: ${params?.selfLearningMetadata?.name}`);
+            console.log(`associated: ${params?.selfLearningMetadata?.associated}`);
+        }
+        let res = await executeStep(workflow, job, step, ctx, inputs, output, workflows, config);
+        if (res.back) {
+            console.log(`Step requested going back, adjusting execution index.`);
+            i = Math.max(0, i - 1); // Go back 1 step, but don't go below 0
+        }
+        else {
+            i++; // Only increment if not going back
+        }
+    }
+    return output;
+}
+export async function executeStep(workflow, job, step, ctx, inputs, output, workflows, config, maxAttempts = 4) {
+    console.log(`------- inputs ---------`);
+    console.log(inputs);
+    // Implementation for executing a single step
+    /*
+    {
+            action: 'move_interact',
+            mediaType: 'image',
+            mediaUrl: 'https://firebasestorage.googleapis.com/v0/b/flowframe-auto.firebasestorage.app/o/users%2Fl7YvrqxTiogIlujkfp9oA6YMAvg2%2Fmedia%2F1754341845667_Screenshot_2025-08-04_at_4.10.26_PM.png?alt=media&token=25089088-56c3-4512-a405-c9e43bfdb7de',
+            timestampms: 2000,
+            type: 'move-interact',
+            id: 'c1fd8fe5-1f9e-42b9-a7e7-f2837015b655',
+            description: 'Move mouse and interact with elements',
+            params: {
+                    targetX: 0.9226277372262773,
+                    action: 'scroll',
+                    generateAnnotations: true,
+                    scrollAmount: 1000,
+                    annotation: {
+                        targetY: 1686,
+                        imageHeight: 2658,
+                        levels: [Array],
+                        stepId: 'c1fd8fe5-1f9e-42b9-a7e7-f2837015b655',
+                        targetX: 3794,
+                        imageWidth: 4112
+                    },
+                    targetY: 0.6343778671748183,
+                    singleLevel: true,
+                    noLevel: false,
+                    className: 'target',
+                    levelReduction: 0.25,
+                    minPixelSize: 20,
+                    rootPercentage: 0.25,
+                    actualMediaSize: { height: 2658, width: 4112 }
+            }
+        }
+   */
+    console.log(`Executing step action: ${step.action} with params:`, step.params);
+    if (step?.params?.smart) {
+        // Smart parameter handling can be implemented here
+        /* "smart": {
+            "cropHeight": 50,
+            "text": "lie, electronic",
+            "hasText": true,
+            "cropWidth": 140,
+            "hasIcon": false,
+            "image": "data:image/png;base64,iVBORw0KGgoAAAANSUh"
+        }*/
+        let grade = 0;
+        let maxAttempts = 5;
+        try {
+            do {
+                grade = await evaluateSmartly(inputs, step);
+                console.log(`Grade for smart analysis: ${grade}`);
+                if (!grade)
+                    await pause(() => { }, 1000);
+            } while (grade < 1 && maxAttempts-- > 0);
+            if (grade > 0) {
+                console.log(`Smart analysis passed with grade: ${grade}, proceeding with step action.`);
+            }
+            else {
+                console.log(`Smart analysis failed after multiple attempts, skipping step action.`);
+                let message = '';
+                if (step.params.smart.hasText) {
+                    message += `Expected text: ${step.params.smart.text}`;
+                }
+                if (step.params.smart.hasIcon) {
+                    message += `Expected icon presence: ${step.params.smart.hasIcon}`;
+                }
+                if (!step.params.validateWithSmartSkippable) {
+                    throw new SmartError(`Smart analysis failed for step ${step.id}`, { details: message });
+                }
+            }
+        }
+        catch (e) {
+            console.log(`Error during smart analysis: ${e}, skipping step action.`);
+        }
+    }
+    if (true) {
+        await performAction();
+    }
+    else {
+        await recordTransition({
+            takeScreenshot: async () => {
+                let filename = `step_${step.id}_before_${Date.now()}.png`;
+                await captureScreenshot(get(KEYS.screen_shot_folder), filename);
+                return path.join(get(KEYS.screen_shot_folder), filename);
+            },
+            recognizeScreen: async (imagePath) => {
+                return recognizeScreen(imagePath);
+            },
+            performAction,
+            action: step.action,
+            meta: { stepId: step.id }
+        });
+    }
+    async function performAction() {
+        const SCREEN = robot.getScreenSize();
+        switch (step.action) {
+            case 'pause':
+                console.log('Waiting for', step.params.duration || 1000, 'ms');
+                await pause(() => { }, step.params.duration || 1000);
+                console.log(`Completed wait of ${step.params.duration || 1000} ms`);
+                break;
+            case 'type':
+                {
+                    console.log(`Typing input for step ${step.id}`);
+                    console.log(`Step params:`, step.params);
+                    console.log(`step.params.keys: ${step.params.keys}`);
+                    await pause(() => { }, 1000);
+                    await executeKeyCommands(step.params.keys, inputs, step.params);
+                    await pause(() => { }, 1000);
+                }
+                break;
+            case 'combination_move-interact':
+                {
+                    console.log(`Executing combination move interact for step ${step.id}`);
+                    let count = getNestedProperty(inputs?.inputs || inputs.a, step.params.countProperty);
+                    console.log(`Retrieved count from inputs at ${step.params.countProperty}: ${count}`);
+                    if (count)
+                        count = parseInt(count);
+                    console.log(`count for combination steps: ${count}`);
+                    for (let i = 0; i < (count || step.params.count || 1); i++) {
+                        for (const [idx, subStep] of (step.params.steps || []).entries()) {
+                            console.log(`  Executing sub-step ${idx + 1}:`, subStep);
+                            await executeStep(workflow, job, subStep, ctx, inputs, output, workflows, config, maxAttempts);
+                        }
+                    }
+                    break;
+                }
+            case 'automation':
+                {
+                    let automationName = step.params.automationName;
+                    if (automationName.endsWith('.automation')) {
+                        automationName = automationName.slice(0, -'.automation'.length);
+                    }
+                    const workflow = workflows.find(wf => {
+                        console.log(wf);
+                        return wf.workflow.id === automationName;
+                    });
+                    if (!workflow) {
+                        console.log(`There are ${workflows.length} workflows loaded:`);
+                        console.log(workflows.map(d => `${d.workflow.id}=> ${d.workflow.name}`).join('\n'));
+                        throw new Error(`Workflow not found: ${automationName}`);
+                    }
+                    await executeConfiguration(workflow, config, ctx, inputs, workflows);
+                }
+                break;
+            case 'move_interact':
+                {
+                    if (step?.params?.selfLearningTags?.length) {
+                        let targetFolder = get(KEYS.screen_shot_folder);
+                        if (fs.existsSync(targetFolder)) {
+                            let filename = await captureScreenshot(get(KEYS.screen_shot_folder), `automation_step_${step.id}_${Date.now()}.png`);
+                            let filePath = path.join(targetFolder, filename);
+                            await ingestScreenshot({
+                                imagePath: filePath,
+                                meta: step?.params?.selfLearningMetadata || {},
+                                tags: step?.params?.selfLearningTags || []
+                            });
+                        }
+                        else {
+                            throw new Error(`Screen shot folder does not exist: ${targetFolder}`);
+                        }
+                    }
+                    const SCREEN = robot.getScreenSize();
+                    if (step.params.targetProperty) {
+                        // read from inputs
+                        let target = getNestedProperty(inputs, step.params.targetProperty);
+                        if (target) {
+                            step.params.targetX = target.x;
+                            step.params.targetY = target.y;
+                        }
+                        else {
+                            throw new Error(`Target property ${step.params.targetProperty} not found in inputs`);
+                        }
+                    }
+                    let center = {
+                        left: step.params.targetX * SCREEN.width,
+                        top: step.params.targetY * SCREEN.height,
+                        height: 2,
+                        width: 2
+                    };
+                    await moveMouseDesktop(center);
+                    await pause(() => { }, 1000);
+                    console.log(`********* Performing action: ${step.params.action} *********`);
+                    switch (step.params.action) {
+                        case "click":
+                            await mouseClick();
+                            break;
+                        case "double_click":
+                            await mouseClick();
+                            await pause(() => { }, 100);
+                            await mouseClick();
+                            break;
+                        case 'selectAllAndClear':
+                            {
+                                clearAllText();
+                            }
+                            break;
+                        case "file_modal_selection":
+                            {
+                                // Implementation for file modal selection can go here
+                                let text = (inputs?.inputs || inputs.a)[step.params.filePropertyName];
+                                if (!text) {
+                                    console.warn(`No text found in inputs?.inputs or inputs.a for property name: ${step.params.filePropertyName}`);
+                                    throw new Error(`No text found in inputs?.inputs or inputs.a for property name: ${step.params.filePropertyName}`);
+                                }
+                                if (!fs.existsSync(text)) {
+                                    console.log(`File does not exist at path: ${text}`);
+                                    throw new Error(`File does not exist at path: ${text}`);
+                                }
+                                await fileModalOperate(text);
+                            }
+                            break;
+                        case 'go_back_if':
+                            {
+                                const SCREEN = robot.getScreenSize();
+                                let answer = 'no'; // Declare outside the loop
+                                await pause(() => { }, 5 * 1000);
+                                // Capture the specified area
+                                let area = {
+                                    x: step.params.left * SCREEN.width,
+                                    y: step.params.top * SCREEN.height,
+                                    width: (step.params.right - step.params.left) * SCREEN.width,
+                                    height: (step.params.bottom - step.params.top) * SCREEN.height
+                                };
+                                console.log(`Capturing area: ${JSON.stringify(area)}`);
+                                const base64Image = await captureScreenshotBase64(area);
+                                let prompt = step.params.prompt || "Is the condition met in the image?";
+                                console.log(`Asking question: ${prompt}`);
+                                let model = step.params.llm || 'gpt-4';
+                                let result = await classifyImageQuery(model, prompt, base64Image);
+                                answer = result.answer; // Update the outer scope variable
+                                let { confidence, reason } = result;
+                                console.log(`Answer: ${answer}, Confidence: ${confidence}, Reason: ${reason}`);
+                                console.log(`answer(${answer}) !== 'yes': ${answer !== 'yes'}`);
+                                if (answer === 'yes') {
+                                    console.log(`Condition met, going back.`);
+                                    return { back: true };
+                                }
+                            }
+                            break;
+                        case 'wait_on_area':
+                            {
+                                const SCREEN = robot.getScreenSize();
+                                let answer = 'no'; // Declare outside the loop
+                                const maxWaitTime = (parseInt(`${step.params.maxWaitTime}`) || 60) * 1000 * 60; // default 60 minutes
+                                const interval = parseInt(`${step.params.interval}`) || 5000; // default 5 seconds
+                                const startTime = Date.now();
+                                do {
+                                    // Capture the specified area
+                                    let area = {
+                                        x: step.params.left * SCREEN.width,
+                                        y: step.params.top * SCREEN.height,
+                                        width: (step.params.right - step.params.left) * SCREEN.width,
+                                        height: (step.params.bottom - step.params.top) * SCREEN.height
+                                    };
+                                    console.log(`Capturing area: ${JSON.stringify(area)}`);
+                                    const base64Image = await captureScreenshotBase64(area);
+                                    let prompt = step.params.prompt || "Is the condition met in the image?";
+                                    console.log(`Asking question: ${prompt}`);
+                                    let model = step.params.llm || 'gpt-4';
+                                    let result = await classifyImageQuery(model, prompt, base64Image);
+                                    answer = result.answer; // Update the outer scope variable
+                                    let { confidence, reason } = result;
+                                    console.log(`Answer: ${answer}, Confidence: ${confidence}, Reason: ${reason}`);
+                                    console.log(`answer(${answer}) !== 'yes': ${answer !== 'yes'}`);
+                                    if (answer !== 'yes') {
+                                        await pause(() => { }, interval);
+                                    }
+                                } while (Date.now() - startTime < maxWaitTime && answer !== 'yes');
+                                if (answer === 'no') {
+                                    console.error(`Condition not met within max wait time of ${maxWaitTime} ms`);
+                                }
+                            }
+                            break;
+                        case 'query_image_question':
+                            {
+                                // Capture the specified area
+                                let area = {
+                                    x: step.params.left * SCREEN.width,
+                                    y: step.params.top * SCREEN.height,
+                                    width: (step.params.right - step.params.left) * SCREEN.width,
+                                    height: (step.params.bottom - step.params.top) * SCREEN.height
+                                };
+                                console.log(`Capturing area: ${JSON.stringify(area)}`);
+                                const base64Image = await captureScreenshotBase64(area);
+                                let prompt = step.params.prompt || "Is the condition met in the image?";
+                                console.log(`Asking question: ${prompt}`);
+                                let model = step.params.llm || 'gpt-4';
+                                let result = await generalClassifyImageQuery(model, prompt, base64Image);
+                                console.log(`Answer: ${result}`);
+                                const outputProperty = step.params.property;
+                                if (outputProperty) {
+                                    setNestedProperty(output, outputProperty, result);
+                                }
+                                const areaProperty = step.params.areaProperty;
+                                if (areaProperty) {
+                                    setNestedProperty(output, areaProperty, area);
+                                }
+                                break;
+                            }
+                        case 'find_image_in_image':
+                            {
+                                // Capture the specified area
+                                let area = {
+                                    x: step.params.left * SCREEN.width,
+                                    y: step.params.top * SCREEN.height,
+                                    width: (step.params.right - step.params.left) * SCREEN.width,
+                                    height: (step.params.bottom - step.params.top) * SCREEN.height
+                                };
+                                console.log(`Capturing area: ${JSON.stringify(area)}`);
+                                const base64Image = await captureScreenshotBase64(area);
+                                let templateImageBase64 = step.params.targetImageBase64;
+                                // Add data URL prefix if not present
+                                if (templateImageBase64 && !templateImageBase64.startsWith('data:')) {
+                                    templateImageBase64 = `data:image/png;base64,${templateImageBase64}`;
+                                }
+                                console.log(`Finding template image in captured area.`);
+                                let result = await findImageInImageQuery(step.params.llm || 'gpt-4', templateImageBase64, base64Image);
+                                console.log(`Result: ${JSON.stringify(result)}`);
+                                // Calculate center coordinates relative to SCREEN if box is found
+                                if (result && result.box && result.box.x_center_rel !== undefined && result.box.y_center_rel !== undefined) {
+                                    // The box coordinates are relative to the captured area, so convert to screen-relative coordinates (0-1)
+                                    const absoluteCenterX = area.x + (result.box.x_center_rel * area.width);
+                                    const absoluteCenterY = area.y + (result.box.y_center_rel * area.height);
+                                    const centerX = absoluteCenterX / SCREEN.width;
+                                    const centerY = absoluteCenterY / SCREEN.height;
+                                    result.center = {
+                                        x: centerX,
+                                        y: centerY,
+                                        targetX: centerX,
+                                        targetY: centerY
+                                    };
+                                    console.log(`Calculated center coordinates relative to screen: (${centerX}, ${centerY})`);
+                                }
+                                const outputProperty = step.params.property;
+                                if (outputProperty) {
+                                    setNestedProperty(output, outputProperty, result);
+                                    setNestedProperty(inputs, outputProperty, result);
+                                }
+                                const areaProperty = step.params.areaProperty;
+                                if (areaProperty) {
+                                    setNestedProperty(output, areaProperty, area);
+                                }
+                                let t = {
+                                    "blenderImage": {
+                                        "found": true,
+                                        "confidence": 0.9,
+                                        "box": {
+                                            "x_center_rel": 0.934,
+                                            "y_center_rel": 0.83,
+                                            "width_rel": 0.03,
+                                            "height_rel": 0.15
+                                        },
+                                        "reason": "The Blender icon in the dock matches the query image.",
+                                        "center": {
+                                            "x": 0.9260384615384615,
+                                            "y": 0.9750666666666667
+                                        }
+                                    },
+                                    "blenderArea": {
+                                        "x": 20.56,
+                                        "y": 1134.0800000000002,
+                                        "width": 2016.4615384615383,
+                                        "height": 194.91999999999993
+                                    }
+                                };
+                                break;
+                            }
+                        case 'query_yes_or_no_on_area':
+                            {
+                                let answer = 'no'; // Declare outside the loop
+                                // Capture the specified area
+                                let area = {
+                                    x: step.params.left * SCREEN.width,
+                                    y: step.params.top * SCREEN.height,
+                                    width: (step.params.right - step.params.left) * SCREEN.width,
+                                    height: (step.params.bottom - step.params.top) * SCREEN.height
+                                };
+                                console.log(`Capturing area: ${JSON.stringify(area)}`);
+                                const base64Image = await captureScreenshotBase64(area);
+                                let prompt = step.params.prompt || "Is the condition met in the image?";
+                                console.log(`Asking question: ${prompt}`);
+                                let model = step.params.llm || 'gpt-4';
+                                let result = await classifyImageQuery(model, prompt, base64Image);
+                                answer = result.answer; // Update the outer scope variable
+                                let { confidence, reason } = result;
+                                console.log(`Answer: ${answer}, Confidence: ${confidence}, Reason: ${reason}`);
+                                console.log(`answer(${answer}) !== 'yes': ${answer !== 'yes'}`);
+                                const outputProperty = step.params.property;
+                                if (outputProperty) {
+                                    setNestedProperty(output, outputProperty, answer);
+                                }
+                                const areaProperty = step.params.areaProperty;
+                                if (areaProperty) {
+                                    setNestedProperty(output, areaProperty, area);
+                                }
+                                break;
+                            }
+                        case 'capture_area':
+                            {
+                                // Capture the specified area
+                                let area = {
+                                    x: step.params.left * SCREEN.width,
+                                    y: step.params.top * SCREEN.height,
+                                    width: (step.params.right - step.params.left) * SCREEN.width,
+                                    height: (step.params.bottom - step.params.top) * SCREEN.height
+                                };
+                                console.log(`Capturing area: ${JSON.stringify(area)}`);
+                                const base64Image = await captureScreenshotBase64(area);
+                                // Also store in output if property is specified
+                                const outputProperty = step.params.property;
+                                if (outputProperty) {
+                                    setNestedProperty(output, outputProperty, base64Image);
+                                }
+                                else {
+                                    console.log(`Captured area and stored in inputs.a[${step.id}]`);
+                                }
+                                const areaProperty = step.params.areaProperty;
+                                if (areaProperty) {
+                                    setNestedProperty(output, areaProperty, area);
+                                }
+                            }
+                            break;
+                        case "paste":
+                            {
+                                // Paste from clipboard
+                                let text = (inputs?.inputs || inputs.a)[step.params.textToType];
+                                console.log(`Pasting text: ${text}`);
+                                await pasteText(text);
+                                console.log(`Pasted text successfully.`);
+                            }
+                            break;
+                        case "scroll":
+                            await scroll(0, step.params.scrollAmount || 1000);
+                            break;
+                        case 'type':
+                            {
+                                // await text(step.params.text || '');
+                                let text = (inputs?.inputs || inputs.a)[step.params.textToType] || step.params.text;
+                                console.log(`Typing text: ${text}`);
+                                robot.typeString(text || '');
+                            }
+                            break;
+                        case 'pressEscape':
+                            await mouseClick();
+                            await pressEscapeAsync();
+                            break;
+                        case 'pressAndDrag':
+                            let dragTo = {
+                                left: (step.params.targetX * SCREEN.width) + (((step.params.dragToX) || 0) * SCREEN.width),
+                                top: (step.params.targetY * SCREEN.height) + (((step.params.dragToY) || 0) * SCREEN.height),
+                                height: 2,
+                                width: 2
+                            };
+                            robot.mouseToggle("down");
+                            await pause(() => { }, 500);
+                            await moveMouseDesktop(dragTo, 2000);
+                            await pause(() => { }, 500);
+                            robot.mouseToggle("up");
+                            break;
+                        case 'clickAndCopyText':
+                            await mouseClick();
+                            // Select all agnosticly for all OSs
+                            robot.keyToggle('a', 'down', process.platform === "darwin" ? "command" : "control");
+                            await pause(() => { }, 500);
+                            robot.keyToggle('a', 'up', process.platform === "darwin" ? "command" : "control");
+                            // Copy to clipboard
+                            robot.keyToggle('c', 'down', process.platform === "darwin" ? "command" : "control");
+                            await pause(() => { }, 500);
+                            robot.keyToggle('c', 'up', process.platform === "darwin" ? "command" : "control");
+                            await pause(() => { }, 500);
+                            let clipboard = (await import('clipboardy')).default;
+                            ;
+                            let copiedText = clipboard.readSync();
+                            // Store copied text in inputs for later steps
+                            let outputString = step.params.output || '';
+                            //outputString can be dot delimited keys like "result.text"
+                            setNestedProperty(output, outputString, copiedText);
+                            console.log(`Copied text: ${copiedText}`);
+                            break;
+                        default:
+                            console.log(`Unknown action: ${step.params.action}`);
+                    }
+                }
+                break;
+            case 'switch':
+                {
+                    console.log(`Executing switch step ${step.id}`);
+                    const { cases, defaultCase, switchProperty } = step.params;
+                    // Get the condition value from the condition input
+                    const switchValue = inputs?.a?.[switchProperty];
+                    console.log(`Switch condition value:`, switchValue);
+                    // Find the matching case
+                    let caseStep = null;
+                    if (switchValue !== undefined && cases && cases[switchValue]) {
+                        caseStep = cases[switchValue];
+                        console.log(`Found matching case "${switchValue}" for step ${step.id}`);
+                    }
+                    else if (defaultCase) {
+                        caseStep = defaultCase;
+                        console.log(`No matching case found, using default case for step ${step.id}`);
+                    }
+                    else {
+                        console.log(`No matching case and no default case for step ${step.id}, skipping`);
+                    }
+                    // Execute the case step if found
+                    if (caseStep) {
+                        console.log(`Executing case step:`, caseStep);
+                        await executeStep(workflow, job, caseStep, ctx, inputs, output, workflows, config, maxAttempts);
+                    }
+                }
+                break;
+        }
+    }
+    await pause(() => { }, 500);
+    return {};
+}
+async function evaluateSmartly(inputs, step) {
+    const SCREEN = robot.getScreenSize();
+    if (step.params.targetProperty) {
+        // read from inputs
+        let target = getNestedProperty(inputs, step.params.targetProperty);
+        if (target) {
+            step.params.targetX = target.x;
+            step.params.targetY = target.y;
+        }
+        else {
+            throw new Error(`Target property ${step.params.targetProperty} not found in inputs`);
+        }
+    }
+    console.log(`Smart parameters detected for step ${step.id}:`, step.params.smart);
+    let model = get(KEYS.default_llm_model);
+    let multiModalModel = get(KEYS.default_llm_multi_modal_model);
+    console.log(`Using default LLM model: ${model} and multi-modal model: ${multiModalModel}`);
+    let area = {
+        x: step.params.targetX * SCREEN.width - (step.params.smart.cropWidth / 2),
+        y: step.params.targetY * SCREEN.height - (step.params.smart.cropHeight / 2),
+        width: step.params.smart.cropWidth,
+        height: step.params.smart.cropHeight
+    };
+    console.log(`Capturing area: ${JSON.stringify(area)}`);
+    const base64Image = await captureScreenshotBase64(area);
+    // Compare expected vs actual, and grade.
+    let grade = 0;
+    if (!step.params.validateWithSmart && !step?.params?.validateWithSmartText) {
+        grade = 1;
+    }
+    else {
+        let { parsed } = await analyzeImageForTextAndIcons(base64Image, multiModalModel);
+        console.log(`Parsed smart analysis result:`, parsed);
+        if (step.params.smart.hasText) {
+            if (parsed.hasText) {
+                let lDistance = similarityScore(`${parsed.text}`, `${step.params.smart.text}`);
+                if (lDistance >= (step?.params?.validateWithSmartText * 100)) {
+                    grade += 1;
+                }
+                console.log(`Text similarity score: ${lDistance}, grade: ${grade}`);
+            }
+        }
+        if (step.params.smart.hasIcon) {
+            if (parsed.hasIcon) {
+                grade += 0.5;
+            }
+        }
+    }
+    return grade;
+}
+/**
+ * Convert a detection result to { left, top, width, height }.
+ * @param {Object} result - Object with a `bbox` field.
+ * @param {Object} [opts]
+ * @param {boolean} [opts.useLocal=true] - Prefer local_x1/local_y1 when available.
+ * @param {boolean} [opts.round=false] - Round numbers to `precision` decimals.
+ * @param {number}  [opts.precision=2]
+ */
+function toLTWH(result, { useLocal = true, round = false, precision = 2 } = {}) {
+    if (!result || !result.bbox)
+        throw new Error('Invalid result: missing bbox');
+    const b = result.bbox;
+    const left = (useLocal && 'local_x1' in b) ? b.local_x1 : b.x1;
+    const top = (useLocal && 'local_y1' in b) ? b.local_y1 : b.y1;
+    const width = ('width' in b) ? b.width : (b.x2 - b.x1);
+    const height = ('height' in b) ? b.height : (b.y2 - b.y1);
+    const rect = { left, top, width, height };
+    let factor = 2;
+    if (isWindows()) {
+        factor = 1;
+    }
+    if (round) {
+        const f = n => Number(n.toFixed(precision));
+        rect.left = f(rect.left / factor);
+        rect.top = f(rect.top / factor);
+        rect.width = f(rect.width / factor);
+        rect.height = f(rect.height / factor);
+    }
+    return rect;
+}
+/**
+ * Get the center point of a detection result.
+ * @param {{ bbox: { x1?:number, y1?:number, x2?:number, y2?:number, width?:number, height?:number, local_x1?:number, local_y1?:number, local_x2?:number, local_y2?:number } }} result
+ * @param {"global"|"local"} [space="global"]  Use "local" to prefer local_* coords if present.
+ * @returns {{ x:number, y:number }}
+ */
+function getCenter(result, space = "global") {
+    const b = (result && result.bbox) || {};
+    const F = Number.isFinite;
+    if (space === "local") {
+        if (F(b.local_x1) && F(b.local_x2) && F(b.local_y1) && F(b.local_y2)) {
+            return { x: (b.local_x1 + b.local_x2) / 2, y: (b.local_y1 + b.local_y2) / 2 };
+        }
+        if (F(b.local_x1) && F(b.local_y1) && F(b.width) && F(b.height)) {
+            return { x: b.local_x1 + b.width / 2, y: b.local_y1 + b.height / 2 };
+        }
+        throw new Error("Local coordinates not available on bbox.");
+    }
+    // global (default)
+    if (F(b.x1) && F(b.x2) && F(b.y1) && F(b.y2)) {
+        return { x: (b.x1 + b.x2) / 2, y: (b.y1 + b.y2) / 2 };
+    }
+    if (F(b.x1) && F(b.y1) && F(b.width) && F(b.height)) {
+        return { x: b.x1 + b.width / 2, y: b.y1 + b.height / 2 };
+    }
+    throw new Error("Global coordinates not available on bbox.");
+}
+//# sourceMappingURL=executor.js.map