@mablhq/mabl-cli 2.94.5 → 2.96.5

package/trainer/generation/toolExecutors/RecordClickExecutor.js

@@ -37,10 +37,10 @@ exports.recordClickExecutor = {
                 success: false,
             };
         }
+        const toolParams = payload.toolCall.toolParams;
         if ((0, toolExecutors_1.isFindOneEventMessage)(payload.find) && payload.find.selectOptions) {
-            const elementDescription = payload.toolCall.toolParams?.elementDescription || 'unknown element';
-            const elementBoundingBox = payload.toolCall.toolParams
-                ?.elementBoundingBox;
+            const elementDescription = toolParams.elementDescription || 'unknown element';
+            const elementBoundingBox = toolParams.elementBoundingBox;
             const { errorMessage, applicationStatement } = buildNativeSelectErrorMessage(elementDescription, elementBoundingBox, payload.find.selectOptions);
             return {
                 success: false,
@@ -49,6 +49,6 @@ exports.recordClickExecutor = {
                 applicationInfo: [applicationStatement],
             };
         }
-        return (0, toolExecutors_1.recordStepAndTrackResult)(() => (0, stepCapture_1.recordTrainedStep)(payload.find, payload.contextInfo, true, payload.toolCall.toolParams?.description, stateProvider, eventHandler), CLICK_STEP_WAIT_TIME_MS, stateProvider, eventHandler, logger);
+        return (0, toolExecutors_1.recordStepAndTrackResult)(() => (0, stepCapture_1.recordTrainedStep)(payload.find, payload.contextInfo, true, undefined, stateProvider, eventHandler), CLICK_STEP_WAIT_TIME_MS, stateProvider, eventHandler, logger);
     },
 };

package/trainer/generation/toolExecutors/RecordEnterTextExecutor.js

@@ -26,6 +26,6 @@ exports.recordEnterTextExecutor = {
         else {
             findEvent.inputValue = textToEnter;
         }
-        return (0, toolExecutors_1.recordStepAndTrackResult)(() => (0, stepCapture_1.recordTrainedStep)(findEvent, payload.contextInfo, true, payload.toolCall.toolParams?.description, stateProvider, eventHandler), ENTER_TEXT_STEP_WAIT_TIME_MS, stateProvider, eventHandler, logger);
+        return (0, toolExecutors_1.recordStepAndTrackResult)(() => (0, stepCapture_1.recordTrainedStep)(findEvent, payload.contextInfo, true, undefined, stateProvider, eventHandler), ENTER_TEXT_STEP_WAIT_TIME_MS, stateProvider, eventHandler, logger);
     },
 };

package/trainer/generation/toolExecutors/RecordHoverExecutor.js

@@ -14,6 +14,6 @@ exports.recordHoverExecutor = {
                 success: false,
             };
         }
-        return (0, toolExecutors_1.recordStepAndTrackResult)(() => (0, stepCapture_1.recordTrainedStep)(payload.find, payload.contextInfo, true, payload.toolCall.toolParams?.description, stateProvider, eventHandler), HOVER_STEP_WAIT_TIME_MS, stateProvider, eventHandler, logger);
+        return (0, toolExecutors_1.recordStepAndTrackResult)(() => (0, stepCapture_1.recordTrainedStep)(payload.find, payload.contextInfo, true, undefined, stateProvider, eventHandler), HOVER_STEP_WAIT_TIME_MS, stateProvider, eventHandler, logger);
     },
 };

package/trainer/generation/toolExecutors/RecordStartTaskExecutor.js

@@ -5,12 +5,17 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.recordStartTaskExecutor = void 0;
 exports.recordStartTaskStep = recordStartTaskStep;
-const toolExecutors_1 = require("../toolExecutors");
+exports.recordStartTaskEchoStep = recordStartTaskEchoStep;
 const generation_types_1 = require("../../sharedUtils/generation-types");
+const ExecuteStepGroupStep_1 = __importDefault(require("../../sharedUtils/mablscript/extended/ExecuteStepGroupStep"));
 const EchoStep_1 = __importDefault(require("../../sharedUtils/mablscript/canonical/EchoStep"));
 function recordStartTaskStep(taskDescription) {
-    const steps = [new EchoStep_1.default(`${generation_types_1.START_TASK_PREFIX} ${taskDescription}`)];
-    return { steps, stepsToReplay: steps.length };
+    const stepGroup = new ExecuteStepGroupStep_1.default([], taskDescription);
+    return { steps: [stepGroup], stepsToReplay: 1 };
+}
+function recordStartTaskEchoStep(taskDescription) {
+    const echoStep = new EchoStep_1.default(`${generation_types_1.START_TASK_PREFIX} ${taskDescription}`);
+    return { steps: [echoStep], stepsToReplay: 1 };
 }
 exports.recordStartTaskExecutor = {
     needsFind: () => false,
@@ -20,6 +25,21 @@ exports.recordStartTaskExecutor = {
             logger.error('Record start task executor requires step description');
             return { success: false };
         }
-        return (0, toolExecutors_1.recordStepAndTrackResult)(() => Promise.resolve(recordStartTaskStep(payload.toolCall.toolDescription)), 0, stateProvider, eventHandler, logger);
+        const taskDescription = payload.toolCall.toolDescription;
+        const useStepGroups = stateProvider.hasStepGroupsFeature();
+        if (useStepGroups) {
+            const stepGroup = new ExecuteStepGroupStep_1.default([], taskDescription);
+            await eventHandler.dispatchSteps([stepGroup], {
+                skipActiveGroupInsertion: true,
+            });
+            if (!eventHandler.isInsideStepGroup()) {
+                eventHandler.moveCursorIntoGroup();
+            }
+        }
+        else {
+            const echoStep = new EchoStep_1.default(`${generation_types_1.START_TASK_PREFIX} ${taskDescription}`);
+            await eventHandler.dispatchSteps([echoStep]);
+        }
+        return { success: true };
     },
 };

package/trainer/generation/toolExecutors/ScrollExecutor.js

@@ -4,6 +4,7 @@ exports.scrollExecutor = void 0;
 const mablApi_1 = require("../../../mablApi");
 const utilities_1 = require("../../utilities");
 const contentActions_1 = require("../../background/capture/contentActions");
+const boundingBoxUtils_1 = require("../../sharedUtils/boundingBoxUtils");
 const DEFAULT_SCROLL_WAIT_TIME_MILLISECONDS = 1000;
 exports.scrollExecutor = {
     needsFind: () => false,
@@ -17,12 +18,15 @@ exports.scrollExecutor = {
                 errorCause: 'IllegalArguments',
             };
         }
+        const internalBoundingBox = toolParams.elementBoundingBox
+            ? (0, boundingBoxUtils_1.convertToolLibraryBoundingBox)(toolParams.elementBoundingBox)
+            : undefined;
         let scrollStatus;
         if (toolParams.direction === 'up') {
-            scrollStatus = await (0, contentActions_1.sendScrollUpMessage)(stateProvider.getTrainingContext(), toolParams.scrollFactor, toolParams.elementTargetPoint, toolParams.elementBoundingBox);
+            scrollStatus = await (0, contentActions_1.sendScrollUpMessage)(stateProvider.getTrainingContext(), toolParams.scrollFactor, toolParams.elementTargetPoint, internalBoundingBox);
         }
         else if (toolParams.direction === 'down') {
-            scrollStatus = await (0, contentActions_1.sendScrollDownMessage)(stateProvider.getTrainingContext(), toolParams.scrollFactor, toolParams.elementTargetPoint, toolParams.elementBoundingBox);
+            scrollStatus = await (0, contentActions_1.sendScrollDownMessage)(stateProvider.getTrainingContext(), toolParams.scrollFactor, toolParams.elementTargetPoint, internalBoundingBox);
         }
         else {
             logger.error(`Unknown scroll direction (${toolParams.direction}) in scroll executor`);

package/trainer/generation/toolExecutors/VisitUrlExecutor.js

@@ -20,6 +20,6 @@ exports.visitUrlExecutor = {
             action: 'record',
             listener: 'visit-url',
             url: toolParams.url,
-        }, contextInfo, true, payload.toolCall.toolParams?.description, stateProvider, eventHandler), 0, stateProvider, eventHandler, logger);
+        }, contextInfo, true, undefined, stateProvider, eventHandler), 0, stateProvider, eventHandler, logger);
     },
 };

package/trainer/generation/toolExecutors/WaitExecutor.js

@@ -15,7 +15,7 @@ exports.waitExecutor = {
                 action: 'record',
                 listener: 'wait',
                 time: toolParams?.waitTime ?? DEFAULT_WAIT_STEP_WAIT_TIME_SECONDS,
-            }, payload.contextInfo ?? (await stateProvider.getActiveContextInfo()), true, payload.toolCall.toolParams?.description, stateProvider, eventHandler), 0, stateProvider, eventHandler, logger);
+            }, payload.contextInfo ?? (await stateProvider.getActiveContextInfo()), true, undefined, stateProvider, eventHandler), 0, stateProvider, eventHandler, logger);
         }
         await (0, utilities_1.sleep)((toolParams?.waitTime ?? DEFAULT_WAIT_STEP_WAIT_TIME_SECONDS) * 1000);
         return {

package/trainer/sharedUtils/boundingBoxUtils.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.convertToolLibraryBoundingBox = convertToolLibraryBoundingBox;
+function convertToolLibraryBoundingBox(toolLibBox) {
+    return {
+        x: toolLibBox.left,
+        y: toolLibBox.top,
+        width: toolLibBox.right - toolLibBox.left,
+        height: toolLibBox.bottom - toolLibBox.top,
+    };
+}

package/trainer/sharedUtils/contextBuilder.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CURSOR_MARKER = void 0;
 exports.parseTaskFromEchoStep = parseTaskFromEchoStep;
+exports.parseTaskFromStep = parseTaskFromStep;
 exports.convertStepToSuccessContext = convertStepToSuccessContext;
 exports.getSupportedTools = getSupportedTools;
 exports.makeStepDescriptionForStartTask = makeStepDescriptionForStartTask;
@@ -13,6 +14,12 @@ function parseTaskFromEchoStep(step) {
         ? step.message.slice(generation_types_1.START_TASK_PREFIX.length).trim()
         : undefined;
 }
+function parseTaskFromStep(step) {
+    if ((0, mablscript_step_types_1.isStepGroupStep)(step)) {
+        return step.description;
+    }
+    return undefined;
+}
 function convertStepToSuccessContext(step) {
     if (!step) {
         return undefined;

package/trainer/sharedUtils/mablscript/extended/ExecuteStepGroupStep.js

@@ -49,6 +49,9 @@ class ExecuteStepGroupStep extends BaseStep_1.default {
         });
         return [...new Set(variables)];
     }
+    willGenerateMablscript() {
+        return false;
+    }
 }
 exports.default = ExecuteStepGroupStep;
 function humanizeExecuteStepGroupStep(step) {

package/trainer/sharedUtils/messageConversion.js

@@ -4,6 +4,7 @@ exports.createUserTextMessage = createUserTextMessage;
 exports.createUserFileMessage = createUserFileMessage;
 exports.createToolOutputMessagePart = createToolOutputMessagePart;
 exports.createToolOutputMessage = createToolOutputMessage;
+exports.createToolOutputFromResult = createToolOutputFromResult;
 exports.extractToolCallsFromMessages = extractToolCallsFromMessages;
 exports.filterClientSideToolCalls = filterClientSideToolCalls;
 exports.filterToolCallsWithoutOutputs = filterToolCallsWithoutOutputs;
@@ -85,6 +86,36 @@ function createToolOutputMessage(toolOutputMessageParts) {
         parts: toolOutputMessageParts,
     };
 }
+function createToolOutputFromResult(params) {
+    const { toolCallId, toolName, wrappedResult } = params;
+    const { result, isError } = wrappedResult;
+    const contentItems = [];
+    contentItems.push({
+        type: 'json',
+        json: result.toolResponse,
+    });
+    if (result.imageContentItems) {
+        for (const img of result.imageContentItems) {
+            if (img.type === 'image-data' && img.image_data && img.media_type) {
+                const imageDataContentItem = {
+                    type: 'image-data',
+                    image_data: img.image_data,
+                    media_type: img.media_type,
+                };
+                contentItems.push(imageDataContentItem);
+            }
+        }
+    }
+    const output = isError
+        ? { type: 'error-content', error_content: contentItems }
+        : { type: 'content', content: contentItems };
+    return {
+        type: 'tool_output',
+        tool_name: toolName,
+        tool_call_id: toolCallId,
+        output,
+    };
+}
 function extractToolCallsFromMessages(messages, logger) {
     const modelMessages = messages.filter((m) => m.role === 'model');
     return modelMessages.flatMap((m) => (m.parts ?? [])