gpt-driver-node 1.0.1 → 1.0.3

package/dist/index.cjs

@@ -22,6 +22,15 @@ function buildUrl(base, extraPath) {
   }
   return `${baseUrl}${extraPath}`;
 }
+const getImageDimensions = async (base64) => {
+  const base64Data = base64.replace(/^data:image\/\w+;base64,/, "");
+  const buffer = Buffer.from(base64Data, "base64");
+  const metadata = await sharp(buffer).metadata();
+  if (!metadata.width || !metadata.height) {
+    throw new Error("Unable to get image dimensions");
+  }
+  return { width: metadata.width, height: metadata.height };
+};
 
 const colors = {
   reset: "\x1B[0m",
@@ -107,6 +116,15 @@ const SavableTypeStepSchema = SavableStepBaseSchema.extend({
   type: zod.z.literal("type"),
   text: zod.z.string()
 });
+const SavableSwipeStepSchema = SavableStepBaseSchema.extend({
+  type: zod.z.literal("swipe"),
+  direction: zod.z.enum(["left", "right", "up", "down"]),
+  x1: zod.z.number().optional(),
+  y1: zod.z.number().optional(),
+  x2: zod.z.number().optional(),
+  y2: zod.z.number().optional(),
+  duration: zod.z.number().optional().default(500)
+});
 const SavableScrollStepSchema = SavableStepBaseSchema.extend({
   type: zod.z.literal("scroll"),
   direction: zod.z.enum(["up", "down"])
@@ -142,6 +160,8 @@ const SavableStepSchema = zod.z.discriminatedUnion("type", [
   // type: 'assert'
   SavableTypeStepSchema,
   // type: 'type'
+  SavableSwipeStepSchema,
+  // type: 'swipe'
   SavableScrollStepSchema,
   // type: 'scroll'
   SavableZoomStepSchema,
@@ -160,6 +180,14 @@ const SavableTestStoreSchema = zod.z.object({
   steps: zod.z.array(SavableStepSchema),
   params: zod.z.record(zod.z.string(), zod.z.string()).optional()
 });
+const VariablesSchema = zod.z.record(zod.z.string(), zod.z.string()).optional().default({});
+const ConfigSchema = zod.z.object({
+  testDir: zod.z.string(),
+  driver: zod.z.string(),
+  port: zod.z.number(),
+  apiKey: zod.z.string(),
+  variables: VariablesSchema
+});
 
 const CACHE_SERVER_URL = "https://cache.mobileboost.io";
 const GPT_DRIVER_BASE_URL = "https://api.mobileboost.io";
@@ -419,6 +447,30 @@ function isScrollCommand(cmd) {
 function isTypeCommand(cmd) {
   return cmd.startsWith("type:");
 }
+function isSlideCommand(cmd) {
+  return cmd.startsWith("slide");
+}
+function parseSlideCommand(cmd) {
+  const slideMatch = cmd.match(
+    /slide\s+(up|down|left|right)\s+(\d+)%(?::\s*[^;]*)?;(\d+);(\d+)/i
+  );
+  if (!slideMatch) {
+    return null;
+  }
+  const extractedDirection = slideMatch[1].toLowerCase();
+  const directionMap = {
+    down: "up",
+    up: "down",
+    left: "right",
+    right: "left"
+  };
+  return {
+    direction: directionMap[extractedDirection],
+    percentage: parseInt(slideMatch[2], 10),
+    startX: parseInt(slideMatch[3], 10),
+    startY: parseInt(slideMatch[4], 10)
+  };
+}
 
 async function executeSmartLoop(ctx, params) {
   const maxCacheAttempts = Math.floor(CACHE_RETRY_MS / CACHE_CHECK_INTERVAL_MS);
@@ -434,6 +486,8 @@ async function executeSmartLoop(ctx, params) {
       let screenshot = "";
       let commands = [];
       let isCacheHit = false;
+      const firstScreenshot = await ctx.getScreenshot();
+      const screenshotResolution = await getImageDimensions(firstScreenshot);
       for (let attempt = 0; attempt < maxCacheAttempts; attempt++) {
         screenshot = await ctx.getScreenshot();
         const sizeInBytes = screenshot.length * 0.75;
@@ -446,7 +500,7 @@ async function executeSmartLoop(ctx, params) {
             stepNumber: params.stepNumber,
             stepDescription: params.description,
             screenshot,
-            screenResolution: ctx.screenSize,
+            screenResolution: screenshotResolution,
             highestUsedIndex: lastCacheIndex,
             platform: ctx.platform,
             filepath: params.filepath
@@ -517,7 +571,12 @@ async function executeSmartLoop(ctx, params) {
           const coords = parseTapCoordinates(cmd);
           if (coords) {
             globalLogger.debug(`[SmartLoop] Executing tap at (${coords.x}, ${coords.y})`);
-            await ctx.performTap(coords.x, coords.y);
+            await ctx.performTap(
+              coords.x,
+              coords.y,
+              screenshotResolution.width,
+              screenshotResolution.height
+            );
             actionExecuted = true;
           }
         } else if (isWaitCommand(cmd)) {
@@ -531,7 +590,44 @@ async function executeSmartLoop(ctx, params) {
           const direction = parseScrollDirection(cmd);
           if (direction) {
             globalLogger.debug(`[SmartLoop] Scrolling ${direction}`);
-            await ctx.performScroll(direction);
+            await ctx.performSwipe({ direction });
+            actionExecuted = true;
+          }
+        } else if (isSlideCommand(cmd)) {
+          const slideParams = parseSlideCommand(cmd);
+          if (slideParams) {
+            const { direction, percentage, startX, startY } = slideParams;
+            const isVertical = direction === "up" || direction === "down";
+            const distance = Math.round(
+              (isVertical ? screenshotResolution.height : screenshotResolution.width) * (percentage / 100)
+            );
+            let endX = startX;
+            let endY = startY;
+            switch (direction) {
+              case "up":
+                endY = startY + distance;
+                break;
+              case "down":
+                endY = startY - distance;
+                break;
+              case "left":
+                endX = startX - distance;
+                break;
+              case "right":
+                endX = startX + distance;
+                break;
+            }
+            globalLogger.debug(`[SmartLoop] Sliding ${direction} ${percentage}% from (${startX}, ${startY}) to (${endX}, ${endY})`);
+            await ctx.performSwipe({
+              direction,
+              x1: startX,
+              y1: startY,
+              x2: endX,
+              y2: endY,
+              screenshotWidth: screenshotResolution.width,
+              screenshotHeight: screenshotResolution.height,
+              duration: 500
+            });
             actionExecuted = true;
           }
         } else if (isTypeCommand(cmd)) {
@@ -559,7 +655,7 @@ async function executeSmartLoop(ctx, params) {
               stepNumber: params.stepNumber,
               stepDescription: params.description,
               executionData: currentExecutionData,
-              screenResolution: ctx.screenSize,
+              screenResolution: screenshotResolution,
               platform: ctx.platform,
               filepath: params.filepath
             });
@@ -604,7 +700,6 @@ class GptDriver {
     });
   }
   apiKey;
-  organisationId;
   gptDriverSessionId;
   gptDriverBaseUrl;
   appiumSessionConfig;
@@ -616,6 +711,8 @@ class GptDriver {
   buildId;
   testId;
   step_number = 1;
+  organisationId;
+  configFilePath;
   // Smart loop state - maintains action history across steps for context
   globalActionHistory = [];
   /**
@@ -640,11 +737,12 @@ class GptDriver {
   constructor(config) {
     this.testId = config.testId;
     this.apiKey = config.apiKey;
-    this.organisationId = config.organisationId;
     this.buildId = config.buildId;
     this.useGptDriverCloud = config.useGptDriverCloud;
     this.gptDriverBaseUrl = GPT_DRIVER_BASE_URL;
     this.cachingMode = config.cachingMode ?? "NONE";
+    this.organisationId = config.organisationId;
+    this.configFilePath = config.configFilePath;
     if (config.useGptDriverCloud) {
       if (config.serverConfig.device?.platform == null) {
         throw new Error("Platform is missing. Please specify the platform when using GPTDriver Cloud.");
@@ -830,11 +928,10 @@ class GptDriver {
     return {
       apiKey: this.apiKey,
       platform: this.appiumSessionConfig?.platform,
-      screenSize: this.appiumSessionConfig.size,
       globalActionHistory: this.globalActionHistory,
-      getScreenshot: () => this.getScreenshot(this.appiumSessionConfig),
-      performTap: (x, y) => this.performTap(x, y),
-      performScroll: (direction) => this.performScroll(direction),
+      getScreenshot: () => this.getScreenshot(this.appiumSessionConfig, false),
+      performTap: (x, y, screenshotWidth, screenshotHeight) => this.performTap(x, y, screenshotWidth, screenshotHeight),
+      performSwipe: (params) => this.performSwipe(params),
       performType: (text) => this.performType(text),
       logCodeExecution: async (screenshot, command) => this.logCodeExecution(screenshot, command),
       organisationId: this.organisationId
@@ -887,15 +984,31 @@ class GptDriver {
   /**
    * Performs a tap action at the specified coordinates.
    */
-  async performTap(x, y) {
+  async performTap(x, y, screenshotWidth, screenshotHeight) {
     const client = await this.getWdioClient();
+    const platform = this.appiumSessionConfig?.platform;
+    const { width: w, height: h } = this.appiumSessionConfig?.size ?? { width: 1080, height: 1920 };
+    const scaled = this.scaleForIOS(
+      x,
+      y,
+      platform,
+      screenshotWidth,
+      screenshotHeight,
+      w,
+      h
+    );
+    const clampedX = this.clamp(scaled.x, 0, w - 1);
+    const clampedY = this.clamp(scaled.y, 0, h - 1);
+    globalLogger.debug(
+      `[Tap] Platform: ${platform}, Input: ${x},${y}, Window: ${w}x${h}, Final: ${clampedX},${clampedY}`
+    );
     await client.performActions([
       {
         type: "pointer",
         id: "finger1",
         parameters: { pointerType: "touch" },
         actions: [
-          { type: "pointerMove", duration: 0, x, y },
+          { type: "pointerMove", duration: 0, x: clampedX, y: clampedY },
           { type: "pointerDown", button: 0 },
           { type: "pause", duration: 100 },
           { type: "pointerUp", button: 0 }
@@ -905,25 +1018,125 @@ class GptDriver {
   }
   async performType(text) {
     const client = await this.getWdioClient();
-    await client.keys(text.split(""));
+    const platform = this.appiumSessionConfig?.platform;
+    if (platform === "iOS") {
+      const actions = text.split("").flatMap((char) => [
+        { type: "keyDown", value: char },
+        { type: "keyUp", value: char }
+      ]);
+      await client.performActions([
+        {
+          type: "key",
+          id: "keyboard",
+          actions
+        }
+      ]);
+    } else {
+      await client.keys(text.split(""));
+    }
   }
-  async performScroll(direction) {
+  clamp(value, min, max) {
+    return Math.max(min, Math.min(max, value));
+  }
+  scaleForIOS(x, y, platform, screenshotWidth, screenshotHeight, windowWidth, windowHeight) {
+    if (platform !== "iOS" || !screenshotWidth || !screenshotHeight || !windowWidth || !windowHeight) {
+      return { x: Math.round(x), y: Math.round(y) };
+    }
+    const scaleX = windowWidth / screenshotWidth;
+    const scaleY = windowHeight / screenshotHeight;
+    return {
+      x: Math.round(x * scaleX),
+      y: Math.round(y * scaleY)
+    };
+  }
+  async performSwipe(params) {
     const client = await this.getWdioClient();
-    const w = this.appiumSessionConfig?.size?.width ?? 1080;
-    const h = this.appiumSessionConfig?.size?.height ?? 1920;
-    const x = Math.round(w / 2);
-    const startY = direction === "down" ? Math.round(h * 0.8) : Math.round(h * 0.2);
-    const endY = direction === "down" ? Math.round(h * 0.2) : Math.round(h * 0.8);
+    const platform = this.appiumSessionConfig?.platform;
+    const { width: w, height: h } = this.appiumSessionConfig?.size ?? { width: 1080, height: 1920 };
+    const defaultStartX = w / 2;
+    let defaultStartY;
+    if (params.direction === "down") {
+      defaultStartY = h * 0.75;
+    } else if (params.direction === "up") {
+      defaultStartY = h * 0.25;
+    } else {
+      defaultStartY = h / 2;
+    }
+    let startX;
+    let startY;
+    if (params.x1 !== void 0 || params.y1 !== void 0) {
+      const scaled = this.scaleForIOS(
+        params.x1 ?? defaultStartX,
+        params.y1 ?? defaultStartY,
+        platform,
+        params.screenshotWidth,
+        params.screenshotHeight,
+        w,
+        h
+      );
+      startX = scaled.x;
+      startY = scaled.y;
+    } else {
+      startX = Math.round(defaultStartX);
+      startY = Math.round(defaultStartY);
+    }
+    startX = this.clamp(startX, 0, w - 1);
+    startY = this.clamp(startY, 0, h - 1);
+    let endX;
+    let endY;
+    if (params.x2 !== void 0 || params.y2 !== void 0) {
+      const scaled = this.scaleForIOS(
+        params.x2 ?? startX,
+        params.y2 ?? startY,
+        platform,
+        params.screenshotWidth,
+        params.screenshotHeight,
+        w,
+        h
+      );
+      endX = scaled.x;
+      endY = scaled.y;
+    } else {
+      const deltaX = Math.round(w * 0.5);
+      const deltaY = Math.round(h * 0.5);
+      switch (params.direction) {
+        case "left":
+          endX = Math.max(0, startX - deltaX);
+          endY = startY;
+          break;
+        case "right":
+          endX = Math.min(w - 1, startX + deltaX);
+          endY = startY;
+          break;
+        case "up":
+          endX = startX;
+          endY = Math.min(h - 1, startY + deltaY);
+          break;
+        case "down":
+          endX = startX;
+          endY = Math.max(0, startY - deltaY);
+          break;
+        default:
+          endX = startX;
+          endY = startY;
+      }
+    }
+    endX = this.clamp(endX, 0, w - 1);
+    endY = this.clamp(endY, 0, h - 1);
+    globalLogger.debug(
+      `[Swipe] Platform: ${platform}, Direction: ${params.direction}, Start: ${startX},${startY}, End: ${endX},${endY}`
+    );
+    const duration = params.duration ?? 500;
     await client.performActions([
       {
         type: "pointer",
         id: "finger1",
         parameters: { pointerType: "touch" },
         actions: [
-          { type: "pointerMove", duration: 0, x, y: startY },
+          { type: "pointerMove", duration: 0, x: startX, y: startY },
           { type: "pointerDown", button: 0 },
           { type: "pause", duration: 100 },
-          { type: "pointerMove", duration: 500, x, y: endY },
+          { type: "pointerMove", duration, x: endX, y: endY },
           { type: "pointerUp", button: 0 }
         ]
       }
@@ -942,17 +1155,17 @@ class GptDriver {
       if (found) {
         return;
       }
-      await this.performScroll(direction);
+      await this.performSwipe({ direction });
       await this._delay(500);
     }
     throw new Error(`scrollUntil target not found after ${max} scroll(s)`);
   }
-  async getScreenshot(appiumSessionConfig) {
+  async getScreenshot(appiumSessionConfig, shouldScale = true) {
     globalLogger.debug("Capturing screenshot...");
     const url = buildUrl(this.appiumSessionConfig.serverUrl, `/session/${this.appiumSessionConfig.id}/screenshot`);
     const screenshotResponse = await axios.get(url);
     let screenshot = await screenshotResponse.data.value;
-    if (appiumSessionConfig.platform === "iOS") {
+    if (appiumSessionConfig.platform === "iOS" && shouldScale) {
       globalLogger.debug(`Resizing iOS screenshot to ${appiumSessionConfig.size.width}x${appiumSessionConfig.size.height}`);
       const imageBuffer = Buffer.from(screenshot, "base64");
       const transformedImage = await sharp(imageBuffer).resize(appiumSessionConfig.size.width, appiumSessionConfig.size.height).toBuffer();
@@ -1005,86 +1218,12 @@ class GptDriver {
       await this.gptHandler(command);
     }
   }
-  /**
-   * Executes a specified command within the WebDriver session with configurable caching options.
-   *
-   * This is the recommended method for executing commands. It provides fine-grained control over
-   * caching behavior, allowing you to optimize performance and costs for repetitive test scenarios.
-   *
-   * If an `appiumHandler` is provided, it will be invoked with the WebDriver instance to perform
-   * the command-specific operations. After executing the handler, the executed commands get logged
-   * on the GPTDriver servers. If the handler execution fails or no handler is provided, the command
-   * gets executed by the GPTDriver using natural language processing.
-   *
-   * @param {Object} params - The execution parameters
-   * @param {string} params.command - The natural language command to be executed by the GPTDriver.
-   *                                  Examples: "Click the login button", "Enter 'test@example.com' in the email field"
-   * @param {AppiumHandler} [params.appiumHandler] - An optional function that processes Appium-specific commands.
-   *                                                  If provided, this handler is executed instead of calling
-   *                                                  the GPTDriver API. Useful for performance optimization when
-   *                                                  you know the exact Appium commands to execute.
-   * @param {CachingMode} [params.cachingMode] - Controls how the GPTDriver caches this command execution.
-   *                                             If not specified, uses the global caching mode set in the constructor.
-   *                                             Options:
-   *                                             - "NONE"
-   *                                             - "FULL_SCREEN"
-   *                                             - "INTERACTION_REGION"
-   * @param {boolean} [params.useSmartLoop] - If true, uses the smart loop execution (Cache -> AI -> Execute -> Populate)
-   *                                          which optimizes execution by checking cache first and populating it after.
-   *                                          Default: false (uses legacy gptHandler)
-   *
-   * @returns {Promise<void>} A promise that resolves when the command execution is complete.
-   *
-   * @throws {Error} If an error occurs during the execution of the Appium handler or while processing
-   *                 the command by the GPTDriver.
-   *
-   * @example
-   * // Basic usage with natural language (no caching)
-   * await driver.aiExecute({
-   *   command: "Click the submit button"
-   * });
-   *
-   * @example
-   * // Full screen caching for repetitive navigation on similar screens
-   * await driver.aiExecute({
-   *   command: "Navigate to the settings page",
-   *   cachingMode: "FULL_SCREEN"
-   * });
-   *
-   * @example
-   * // Interaction region caching for repeated actions on the same button
-   * await driver.aiExecute({
-   *   command: "Click the login button",
-   *   cachingMode: "INTERACTION_REGION"
-   * });
-   *
-   * @example
-   * // With custom Appium handler as fallback
-   * await driver.aiExecute({
-   *   command: "Click the login button",
-   *   appiumHandler: async (driver) => {
-   *     const loginBtn = await driver.$('~loginButton');
-   *     await loginBtn.click();
-   *   },
-   *   cachingMode: "INTERACTION_REGION"
-   * });
-   *
-   * @example
-   * // Force fresh execution for dynamic content
-   * await driver.aiExecute({
-   *   command: "Verify the current timestamp",
-   *   cachingMode: "NONE"
-   * });
-   *
-   * @example
-   * // Using smart loop for optimized caching
-   * await driver.aiExecute({
-   *   command: "Click the login button",
-   *   useSmartLoop: true,
-   *   cachingMode: "FULL_SCREEN"
-   * });
-   */
-  async aiExecute({ command, appiumHandler, cachingMode, useSmartLoop = false }) {
+  async aiExecute(commandOrOptions, options) {
+    const command = typeof commandOrOptions === "string" ? commandOrOptions : commandOrOptions.command;
+    const opts = typeof commandOrOptions === "string" ? options : commandOrOptions;
+    const appiumHandler = opts?.appiumHandler;
+    const cachingMode = opts?.cachingMode;
+    const useSmartLoop = opts?.useSmartLoop ?? false;
     if (!this.appiumSessionStarted) {
       await this.startSession();
     }
@@ -1337,9 +1476,42 @@ class GptDriver {
    */
   async executeFlow(filePath, options) {
     const useSmartLoop = options?.useSmartLoop ?? false;
+    const configFilePath = this.configFilePath;
+    let baseDir;
+    let absolutePath;
+    if (configFilePath) {
+      let raw2;
+      try {
+        raw2 = await node_fs.promises.readFile(configFilePath, "utf-8");
+      } catch (e) {
+        const msg = `Failed to read file at ${configFilePath}: ${e?.message ?? e}`;
+        globalLogger.error(msg);
+        throw new Error(msg);
+      }
+      let json2;
+      try {
+        json2 = JSON.parse(raw2);
+      } catch (e) {
+        const msg = `Invalid JSON in flow file ${configFilePath}: ${e?.message ?? e}`;
+        globalLogger.error(msg);
+        throw new Error(msg);
+      }
+      const parsedConfigFile = ConfigSchema.parse(json2);
+      if (path.isAbsolute(parsedConfigFile.testDir)) {
+        baseDir = parsedConfigFile.testDir;
+      } else {
+        baseDir = path.resolve(path.dirname(configFilePath), parsedConfigFile.testDir);
+      }
+      absolutePath = path.isAbsolute(filePath) ? filePath : path.resolve(baseDir, filePath);
+    } else {
+      if (useSmartLoop) {
+        throw new Error("Config file is required when using SmartLoop, please provide the path in the constructor");
+      } else {
+        absolutePath = path.resolve(filePath);
+        baseDir = path.dirname(absolutePath);
+      }
+    }
     globalLogger.info(`Loading flow from file: ${filePath}`);
-    const absolutePath = path.resolve(filePath);
-    const baseDir = path.dirname(absolutePath);
     let raw;
     try {
       raw = await node_fs.promises.readFile(absolutePath, "utf-8");
@@ -1379,7 +1551,7 @@ ${issues}`);
       }
       return val.data;
     };
-    const expandSteps = async (steps, inheritedParams, parentDir, stack) => {
+    const expandSteps = async (steps, inheritedParams, parentDir, stack, currentFilePath) => {
       const out = [];
       for (const step of steps) {
         if (step.type === "fileRef") {
@@ -1393,17 +1565,23 @@ ${issues}`);
           const child = await loadFlow(refPath);
           const mergedParams = { ...inheritedParams, ...step.overrides ?? {} };
           const childDir = path.dirname(refPath);
-          const childExpanded = await expandSteps(child.steps, mergedParams, childDir, [...stack, refKey]);
+          const childRelativePath = path.relative(baseDir, refPath).replace(/^\.\//, "");
+          const childExpanded = await expandSteps(child.steps, mergedParams, childDir, [...stack, refKey], childRelativePath);
           out.push(...childExpanded);
         } else {
-          const resolved = { ...step, __params: { ...inheritedParams } };
+          const resolved = {
+            ...step,
+            __params: { ...inheritedParams },
+            __filepath: currentFilePath
+          };
           out.push(resolved);
         }
       }
       return out;
     };
     const effectiveParams = { ...rootFlow.params ?? {} };
-    const expandedSteps = await expandSteps(rootFlow.steps, effectiveParams, baseDir, [absolutePath]);
+    const rootRelativePath = path.relative(baseDir, absolutePath).replace(/^\.\//, "");
+    const expandedSteps = await expandSteps(rootFlow.steps, effectiveParams, baseDir, [absolutePath], rootRelativePath);
     if (!this.appiumSessionStarted) {
       await this.startSession();
     }
@@ -1412,7 +1590,8 @@ ${issues}`);
     try {
       for (const step of expandedSteps) {
         const params = step.__params ?? effectiveParams;
-        const prefix = `Step #${executed + 1} [${step.type}${step.optional ? ", optional" : ""}]`;
+        const filepath = step.__filepath ?? rootRelativePath;
+        const prefix = `Step #${executed + 1} [${step.type}${step.optional ? ", optional" : ""}] (filepath: ${filepath})`;
         try {
           switch (step.type) {
             case "ai": {
@@ -1423,7 +1602,8 @@ ${issues}`);
                 const result = await executeSmartLoop(ctx, {
                   stepNumber: this.step_number,
                   description: instruction,
-                  instruction
+                  instruction,
+                  filepath
                 });
                 if (!result.success) {
                   throw new Error(result.error || "Smart loop execution failed");
@@ -1445,7 +1625,8 @@ ${issues}`);
                 const result = await executeSmartLoop(ctx, {
                   stepNumber: this.step_number,
                   description,
-                  instruction: description
+                  instruction: description,
+                  filepath
                 });
                 if (!result.success) {
                   throw new Error(result.error || "Smart loop execution failed");
@@ -1468,7 +1649,8 @@ ${issues}`);
                 const result = await executeSmartLoop(ctx, {
                   stepNumber: this.step_number,
                   description,
-                  instruction
+                  instruction,
+                  filepath
                 });
                 if (!result.success) {
                   throw new Error(result.error || "Smart loop execution failed");
@@ -1487,11 +1669,39 @@ ${issues}`);
               this.step_number++;
               break;
             }
-            case "scroll": {
-              globalLogger.info(`${prefix}: Scroll ${step.direction}`);
-              await this.takeScreenshotAndLogCodeExecution(`scroll: direction=${step.direction}`);
-              await this.performScroll(step.direction);
-              this.step_number++;
+            case "scroll":
+            case "swipe": {
+              const description = step.descriptionText ? this.interpolateTemplate(step.descriptionText, params) : void 0;
+              if (description && useSmartLoop) {
+                globalLogger.info(`${prefix}: ${description}`);
+                const ctx = this.createSmartLoopContext();
+                const result = await executeSmartLoop(ctx, {
+                  stepNumber: this.step_number,
+                  description,
+                  instruction: description,
+                  filepath
+                });
+                if (!result.success) {
+                  throw new Error(result.error || "Smart loop execution failed");
+                }
+                this.step_number++;
+              } else {
+                globalLogger.info(`${prefix}: ${step.type} ${step.direction}`);
+                await this.takeScreenshotAndLogCodeExecution(`${step.type}: direction=${step.direction}`);
+                if (step.type === "swipe") {
+                  await this.performSwipe({
+                    direction: step.direction,
+                    x1: step.x1,
+                    y1: step.y1,
+                    x2: step.x2,
+                    y2: step.y2,
+                    duration: step.duration
+                  });
+                } else {
+                  await this.performSwipe({ direction: step.direction });
+                }
+                this.step_number++;
+              }
               break;
             }
             case "zoom": {
@@ -1578,10 +1788,7 @@ ${issues}`);
         for (const appiumCommand of executeResponse.commands) {
           await this.executeCommand(appiumCommand);
         }
-        if (!conditionSucceeded) {
-          globalLogger.debug("Command still in progress, waiting...");
-          await delay(1500);
-        }
+        await delay(1500);
       }
       this.step_number = this.step_number + 1;
       globalLogger.info("Command execution completed successfully");
@@ -1594,8 +1801,7 @@ ${issues}`);
   async executeCommand(command) {
     const firstAction = command.data?.actions?.at(0);
     if (firstAction?.type === "pause" && firstAction.duration != null) {
-      globalLogger.debug(`Pausing for ${firstAction.duration} seconds`);
-      await delay(firstAction * 1e3);
+      await delay(firstAction.duration * 1e3);
     } else if (!this.useGptDriverCloud) {
       const parsedUrl = new URL(command.url);
       parsedUrl.protocol = this.appiumSessionConfig.serverUrl.protocol;