@mablhq/mabl-cli 2.94.5 → 2.94.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mablhq/mabl-cli",
-  "version": "2.94.5",
+  "version": "2.94.10",
   "license": "SEE LICENSE IN LICENSE.txt",
   "description": "The official mabl command line interface tool",
   "main": "index.js",

package/trainer/generation/TestGenerationOrchestrator.js

@@ -7,6 +7,7 @@ exports.TestGenerationOrchestrator = void 0;
 const async_retry_1 = __importDefault(require("async-retry"));
 const mablApi_1 = require("../../mablApi");
 const toolExecutors_1 = require("./toolExecutors");
+const boundingBoxUtils_1 = require("../sharedUtils/boundingBoxUtils");
 const contentActions_1 = require("../background/capture/contentActions");
 const generation_types_1 = require("../sharedUtils/generation-types");
 const messageConversion_1 = require("../sharedUtils/messageConversion");
@@ -252,7 +253,9 @@ class TestGenerationOrchestrator {
             };
             this.logger.info('[runGenerationLoop] Tool call information', JSON.stringify(toolCall, undefined, 2));
             if (toolExecutor.needsFind(toolCall)) {
-                const boundingBox = TestGenerationOrchestrator.buildBoundingBox(toolCall.toolParams?.elementBoundingBox);
+                const boundingBox = toolCall.toolParams?.elementBoundingBox
+                    ? (0, boundingBoxUtils_1.convertToolLibraryBoundingBox)(toolCall.toolParams.elementBoundingBox)
+                    : undefined;
                 if (!boundingBox) {
                     this.logger.error(`Tool ${originalToolCall.name} requires element location but no bounding box was provided by the API`);
                     skippedTools.push({
@@ -452,17 +455,6 @@ class TestGenerationOrchestrator {
                 return true;
         }
     }
-    static buildBoundingBox(responseBoundingBox) {
-        if (!responseBoundingBox) {
-            return undefined;
-        }
-        return {
-            x: responseBoundingBox.left,
-            y: responseBoundingBox.top,
-            width: responseBoundingBox.right - responseBoundingBox.left,
-            height: responseBoundingBox.bottom - responseBoundingBox.top,
-        };
-    }
     async captureToolOutputContext(success, toolName, tabsBefore = []) {
         const { needsScreenshot, needsUrl, needsTabInfo } = (0, toolExecutors_1.toolNeedsPostExecutionContext)(toolName, success);
         const context = {};

package/trainer/generation/toolExecutors/DeleteLastStepsExecutor.js

@@ -6,7 +6,8 @@ exports.deleteLastStepsExecutor = {
     needsFind: () => false,
     getStepType: () => mablApi_1.ExecuteToolName.DeleteLastSteps,
     execute: (payload, _stateProvider, eventHandler, logger) => {
-        const toolParams = payload.toolCall.toolParams;
+        const toolParams = payload.toolCall
+            .toolParams;
         const { stepsToDelete } = toolParams;
         if (typeof stepsToDelete !== 'number' || stepsToDelete <= 0) {
             logger.error('Invalid stepsToDelete for RecordDeleteLastStepsExecutor');

package/trainer/generation/toolExecutors/RecordAssertElementExecutor.js

@@ -9,7 +9,8 @@ exports.recordAssertElementExecutor = {
     needsFind: () => true,
     getStepType: () => 'assert',
     execute: async (payload, stateProvider, eventHandler, logger) => {
-        const toolParams = payload.toolCall.toolParams;
+        const toolParams = payload.toolCall
+            .toolParams;
         if (!payload.find || !payload.contextInfo) {
             logger.error('RecordAssertElementExecutor requires find and contextInfo');
             return {
@@ -37,6 +38,6 @@ exports.recordAssertElementExecutor = {
                 elementAttrProp: 'innerText',
             },
         };
-        return (0, toolExecutors_1.recordStepAndTrackResult)(() => (0, stepCapture_1.recordTrainedStep)(step, payload.contextInfo, true, payload.toolCall.toolParams?.description, stateProvider, eventHandler), 0, stateProvider, eventHandler, logger);
+        return (0, toolExecutors_1.recordStepAndTrackResult)(() => (0, stepCapture_1.recordTrainedStep)(step, payload.contextInfo, true, undefined, stateProvider, eventHandler), 0, stateProvider, eventHandler, logger);
     },
 };

package/trainer/generation/toolExecutors/RecordAssertionExecutor.js

@@ -6,14 +6,9 @@ const generation_types_1 = require("../../sharedUtils/generation-types");
 const ExtractDescriptor_1 = require("../../../mablscript/types/ExtractDescriptor");
 const AssertStep_1 = require("../../../mablscript/steps/AssertStep");
 const stepExecution_1 = require("../../sharedUtils/stepExecution");
-const translateMablscript_1 = require("../../sharedUtils/capture/translateMablscript");
-const mablscriptFind_1 = require("../../../mablscriptFind");
 const stepCapture_1 = require("../../background/capture/stepCapture");
 exports.recordAssertionExecutor = {
-    needsFind: (toolCall) => {
-        const toolParams = toolCall.toolParams;
-        return isAssertToolParams(toolParams) && !!toolParams.elementBoundingBox;
-    },
+    needsFind: () => false,
     getStepType: () => 'assert',
     execute: async (payload, stateProvider, eventHandler, logger) => {
         const toolParams = payload.toolCall.toolParams;
@@ -23,28 +18,14 @@ exports.recordAssertionExecutor = {
                 success: false,
             };
         }
-        const isElementAssert = !!toolParams.elementBoundingBox;
-        if (isElementAssert) {
-            if (!payload.find || !payload.contextInfo) {
-                logger.error('Record assertion tool executor with element bounding box requires find and context info');
-                return {
-                    success: false,
-                };
-            }
-        }
-        if ((0, toolExecutors_1.isFindOneEventMessage)(payload.find)) {
-            delete payload.find.stepType;
-        }
-        const step = (0, toolExecutors_1.isFindOneEventMessage)(payload.find)
-            ? { ...payload.find, listener: 'mouse-select' }
-            : {
-                selectionType: 'viewport',
-                listener: 'viewport-assert',
-                findDescriptor: {
-                    type: 'viewport',
-                },
-                action: 'record',
-            };
+        const step = {
+            selectionType: 'viewport',
+            listener: 'viewport-assert',
+            findDescriptor: {
+                type: 'viewport',
+            },
+            action: 'record',
+        };
         step.assertion = {
             assertionType: 'ai_prompt',
             assertionTarget: 'element',
@@ -54,14 +35,11 @@ exports.recordAssertionExecutor = {
             },
             extractionType: ExtractDescriptor_1.ExtractType.EXTRACT_PROMPT,
             onFailure: AssertStep_1.OnFailure.FailTestAtEnd,
-            selectionType: isElementAssert ? 'element' : 'viewport',
-            elementScreenshot: isElementAssert,
+            selectionType: 'viewport',
+            elementScreenshot: false,
         };
         const contextInfo = payload.contextInfo ?? (await stateProvider.getActiveContextInfo());
-        const findDescriptor = isElementAssert && (0, toolExecutors_1.isFindOneEventMessage)(payload.find)
-            ?
-                (0, translateMablscript_1.createFindDescriptor)(payload.find, new mablscriptFind_1.ElementSelectorStore())
-            : { type: 'viewport' };
+        const findDescriptor = { type: 'viewport' };
         const variables = stateProvider.getVariables();
         const testAssertionResult = await (0, stepExecution_1.runAIAssertion)(eventHandler, variables, findDescriptor, step.assertion, contextInfo.frameId, `${contextInfo.tabId}`);
         step.assertion.aiPrompt.criteria =
@@ -74,7 +52,7 @@ exports.recordAssertionExecutor = {
                 success: false,
             };
         }
-        return (0, toolExecutors_1.recordStepAndTrackResult)(() => (0, stepCapture_1.recordTrainedStep)(step, contextInfo, true, payload.toolCall.toolParams?.description, stateProvider, eventHandler), 0, stateProvider, eventHandler, logger);
+        return (0, toolExecutors_1.recordStepAndTrackResult)(() => (0, stepCapture_1.recordTrainedStep)(step, contextInfo, true, undefined, stateProvider, eventHandler), 0, stateProvider, eventHandler, logger);
     },
 };
 function isAssertToolParams(toolParams) {

package/trainer/generation/toolExecutors/RecordClickExecutor.js

@@ -37,10 +37,10 @@ exports.recordClickExecutor = {
                 success: false,
             };
         }
+        const toolParams = payload.toolCall.toolParams;
         if ((0, toolExecutors_1.isFindOneEventMessage)(payload.find) && payload.find.selectOptions) {
-            const elementDescription = payload.toolCall.toolParams?.elementDescription || 'unknown element';
-            const elementBoundingBox = payload.toolCall.toolParams
-                ?.elementBoundingBox;
+            const elementDescription = toolParams.elementDescription || 'unknown element';
+            const elementBoundingBox = toolParams.elementBoundingBox;
             const { errorMessage, applicationStatement } = buildNativeSelectErrorMessage(elementDescription, elementBoundingBox, payload.find.selectOptions);
             return {
                 success: false,
@@ -49,6 +49,6 @@ exports.recordClickExecutor = {
                 applicationInfo: [applicationStatement],
             };
         }
-        return (0, toolExecutors_1.recordStepAndTrackResult)(() => (0, stepCapture_1.recordTrainedStep)(payload.find, payload.contextInfo, true, payload.toolCall.toolParams?.description, stateProvider, eventHandler), CLICK_STEP_WAIT_TIME_MS, stateProvider, eventHandler, logger);
+        return (0, toolExecutors_1.recordStepAndTrackResult)(() => (0, stepCapture_1.recordTrainedStep)(payload.find, payload.contextInfo, true, undefined, stateProvider, eventHandler), CLICK_STEP_WAIT_TIME_MS, stateProvider, eventHandler, logger);
     },
 };

package/trainer/generation/toolExecutors/RecordEnterTextExecutor.js

@@ -26,6 +26,6 @@ exports.recordEnterTextExecutor = {
         else {
             findEvent.inputValue = textToEnter;
         }
-        return (0, toolExecutors_1.recordStepAndTrackResult)(() => (0, stepCapture_1.recordTrainedStep)(findEvent, payload.contextInfo, true, payload.toolCall.toolParams?.description, stateProvider, eventHandler), ENTER_TEXT_STEP_WAIT_TIME_MS, stateProvider, eventHandler, logger);
+        return (0, toolExecutors_1.recordStepAndTrackResult)(() => (0, stepCapture_1.recordTrainedStep)(findEvent, payload.contextInfo, true, undefined, stateProvider, eventHandler), ENTER_TEXT_STEP_WAIT_TIME_MS, stateProvider, eventHandler, logger);
     },
 };

package/trainer/generation/toolExecutors/RecordHoverExecutor.js

@@ -14,6 +14,6 @@ exports.recordHoverExecutor = {
                 success: false,
             };
         }
-        return (0, toolExecutors_1.recordStepAndTrackResult)(() => (0, stepCapture_1.recordTrainedStep)(payload.find, payload.contextInfo, true, payload.toolCall.toolParams?.description, stateProvider, eventHandler), HOVER_STEP_WAIT_TIME_MS, stateProvider, eventHandler, logger);
+        return (0, toolExecutors_1.recordStepAndTrackResult)(() => (0, stepCapture_1.recordTrainedStep)(payload.find, payload.contextInfo, true, undefined, stateProvider, eventHandler), HOVER_STEP_WAIT_TIME_MS, stateProvider, eventHandler, logger);
     },
 };

package/trainer/generation/toolExecutors/ScrollExecutor.js

@@ -4,6 +4,7 @@ exports.scrollExecutor = void 0;
 const mablApi_1 = require("../../../mablApi");
 const utilities_1 = require("../../utilities");
 const contentActions_1 = require("../../background/capture/contentActions");
+const boundingBoxUtils_1 = require("../../sharedUtils/boundingBoxUtils");
 const DEFAULT_SCROLL_WAIT_TIME_MILLISECONDS = 1000;
 exports.scrollExecutor = {
     needsFind: () => false,
@@ -17,12 +18,15 @@ exports.scrollExecutor = {
                 errorCause: 'IllegalArguments',
             };
         }
+        const internalBoundingBox = toolParams.elementBoundingBox
+            ? (0, boundingBoxUtils_1.convertToolLibraryBoundingBox)(toolParams.elementBoundingBox)
+            : undefined;
         let scrollStatus;
         if (toolParams.direction === 'up') {
-            scrollStatus = await (0, contentActions_1.sendScrollUpMessage)(stateProvider.getTrainingContext(), toolParams.scrollFactor, toolParams.elementTargetPoint, toolParams.elementBoundingBox);
+            scrollStatus = await (0, contentActions_1.sendScrollUpMessage)(stateProvider.getTrainingContext(), toolParams.scrollFactor, toolParams.elementTargetPoint, internalBoundingBox);
         }
         else if (toolParams.direction === 'down') {
-            scrollStatus = await (0, contentActions_1.sendScrollDownMessage)(stateProvider.getTrainingContext(), toolParams.scrollFactor, toolParams.elementTargetPoint, toolParams.elementBoundingBox);
+            scrollStatus = await (0, contentActions_1.sendScrollDownMessage)(stateProvider.getTrainingContext(), toolParams.scrollFactor, toolParams.elementTargetPoint, internalBoundingBox);
         }
         else {
             logger.error(`Unknown scroll direction (${toolParams.direction}) in scroll executor`);

package/trainer/generation/toolExecutors/VisitUrlExecutor.js

@@ -20,6 +20,6 @@ exports.visitUrlExecutor = {
             action: 'record',
             listener: 'visit-url',
             url: toolParams.url,
-        }, contextInfo, true, payload.toolCall.toolParams?.description, stateProvider, eventHandler), 0, stateProvider, eventHandler, logger);
+        }, contextInfo, true, undefined, stateProvider, eventHandler), 0, stateProvider, eventHandler, logger);
     },
 };

package/trainer/generation/toolExecutors/WaitExecutor.js

@@ -15,7 +15,7 @@ exports.waitExecutor = {
                 action: 'record',
                 listener: 'wait',
                 time: toolParams?.waitTime ?? DEFAULT_WAIT_STEP_WAIT_TIME_SECONDS,
-            }, payload.contextInfo ?? (await stateProvider.getActiveContextInfo()), true, payload.toolCall.toolParams?.description, stateProvider, eventHandler), 0, stateProvider, eventHandler, logger);
+            }, payload.contextInfo ?? (await stateProvider.getActiveContextInfo()), true, undefined, stateProvider, eventHandler), 0, stateProvider, eventHandler, logger);
         }
         await (0, utilities_1.sleep)((toolParams?.waitTime ?? DEFAULT_WAIT_STEP_WAIT_TIME_SECONDS) * 1000);
         return {

package/trainer/sharedUtils/boundingBoxUtils.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.convertToolLibraryBoundingBox = convertToolLibraryBoundingBox;
+function convertToolLibraryBoundingBox(toolLibBox) {
+    return {
+        x: toolLibBox.left,
+        y: toolLibBox.top,
+        width: toolLibBox.right - toolLibBox.left,
+        height: toolLibBox.bottom - toolLibBox.top,
+    };
+}

package/trainer/sharedUtils/messageConversion.js

@@ -4,6 +4,7 @@ exports.createUserTextMessage = createUserTextMessage;
 exports.createUserFileMessage = createUserFileMessage;
 exports.createToolOutputMessagePart = createToolOutputMessagePart;
 exports.createToolOutputMessage = createToolOutputMessage;
+exports.createToolOutputFromResult = createToolOutputFromResult;
 exports.extractToolCallsFromMessages = extractToolCallsFromMessages;
 exports.filterClientSideToolCalls = filterClientSideToolCalls;
 exports.filterToolCallsWithoutOutputs = filterToolCallsWithoutOutputs;
@@ -85,6 +86,36 @@ function createToolOutputMessage(toolOutputMessageParts) {
         parts: toolOutputMessageParts,
     };
 }
+function createToolOutputFromResult(params) {
+    const { toolCallId, toolName, wrappedResult } = params;
+    const { result, isError } = wrappedResult;
+    const contentItems = [];
+    contentItems.push({
+        type: 'json',
+        json: result.toolResponse,
+    });
+    if (result.imageContentItems) {
+        for (const img of result.imageContentItems) {
+            if (img.type === 'image-data' && img.image_data && img.media_type) {
+                const imageDataContentItem = {
+                    type: 'image-data',
+                    image_data: img.image_data,
+                    media_type: img.media_type,
+                };
+                contentItems.push(imageDataContentItem);
+            }
+        }
+    }
+    const output = isError
+        ? { type: 'error-content', error_content: contentItems }
+        : { type: 'content', content: contentItems };
+    return {
+        type: 'tool_output',
+        tool_name: toolName,
+        tool_call_id: toolCallId,
+        output,
+    };
+}
 function extractToolCallsFromMessages(messages, logger) {
     const modelMessages = messages.filter((m) => m.role === 'model');
     return modelMessages.flatMap((m) => (m.parts ?? [])