gpt-driver-node 1.0.2 → 1.0.3

package/dist/index.cjs

@@ -22,6 +22,15 @@ function buildUrl(base, extraPath) {
   }
   return `${baseUrl}${extraPath}`;
 }
+const getImageDimensions = async (base64) => {
+  const base64Data = base64.replace(/^data:image\/\w+;base64,/, "");
+  const buffer = Buffer.from(base64Data, "base64");
+  const metadata = await sharp(buffer).metadata();
+  if (!metadata.width || !metadata.height) {
+    throw new Error("Unable to get image dimensions");
+  }
+  return { width: metadata.width, height: metadata.height };
+};
 
 const colors = {
   reset: "\x1B[0m",
@@ -107,6 +116,15 @@ const SavableTypeStepSchema = SavableStepBaseSchema.extend({
   type: zod.z.literal("type"),
   text: zod.z.string()
 });
+const SavableSwipeStepSchema = SavableStepBaseSchema.extend({
+  type: zod.z.literal("swipe"),
+  direction: zod.z.enum(["left", "right", "up", "down"]),
+  x1: zod.z.number().optional(),
+  y1: zod.z.number().optional(),
+  x2: zod.z.number().optional(),
+  y2: zod.z.number().optional(),
+  duration: zod.z.number().optional().default(500)
+});
 const SavableScrollStepSchema = SavableStepBaseSchema.extend({
   type: zod.z.literal("scroll"),
   direction: zod.z.enum(["up", "down"])
@@ -142,6 +160,8 @@ const SavableStepSchema = zod.z.discriminatedUnion("type", [
   // type: 'assert'
   SavableTypeStepSchema,
   // type: 'type'
+  SavableSwipeStepSchema,
+  // type: 'swipe'
   SavableScrollStepSchema,
   // type: 'scroll'
   SavableZoomStepSchema,
@@ -427,6 +447,30 @@ function isScrollCommand(cmd) {
 function isTypeCommand(cmd) {
   return cmd.startsWith("type:");
 }
+function isSlideCommand(cmd) {
+  return cmd.startsWith("slide");
+}
+function parseSlideCommand(cmd) {
+  const slideMatch = cmd.match(
+    /slide\s+(up|down|left|right)\s+(\d+)%(?::\s*[^;]*)?;(\d+);(\d+)/i
+  );
+  if (!slideMatch) {
+    return null;
+  }
+  const extractedDirection = slideMatch[1].toLowerCase();
+  const directionMap = {
+    down: "up",
+    up: "down",
+    left: "right",
+    right: "left"
+  };
+  return {
+    direction: directionMap[extractedDirection],
+    percentage: parseInt(slideMatch[2], 10),
+    startX: parseInt(slideMatch[3], 10),
+    startY: parseInt(slideMatch[4], 10)
+  };
+}
 
 async function executeSmartLoop(ctx, params) {
   const maxCacheAttempts = Math.floor(CACHE_RETRY_MS / CACHE_CHECK_INTERVAL_MS);
@@ -442,6 +486,8 @@ async function executeSmartLoop(ctx, params) {
       let screenshot = "";
       let commands = [];
       let isCacheHit = false;
+      const firstScreenshot = await ctx.getScreenshot();
+      const screenshotResolution = await getImageDimensions(firstScreenshot);
       for (let attempt = 0; attempt < maxCacheAttempts; attempt++) {
         screenshot = await ctx.getScreenshot();
         const sizeInBytes = screenshot.length * 0.75;
@@ -454,7 +500,7 @@ async function executeSmartLoop(ctx, params) {
             stepNumber: params.stepNumber,
             stepDescription: params.description,
             screenshot,
-            screenResolution: ctx.screenSize,
+            screenResolution: screenshotResolution,
             highestUsedIndex: lastCacheIndex,
             platform: ctx.platform,
             filepath: params.filepath
@@ -525,7 +571,12 @@ async function executeSmartLoop(ctx, params) {
           const coords = parseTapCoordinates(cmd);
           if (coords) {
             globalLogger.debug(`[SmartLoop] Executing tap at (${coords.x}, ${coords.y})`);
-            await ctx.performTap(coords.x, coords.y);
+            await ctx.performTap(
+              coords.x,
+              coords.y,
+              screenshotResolution.width,
+              screenshotResolution.height
+            );
             actionExecuted = true;
           }
         } else if (isWaitCommand(cmd)) {
@@ -539,7 +590,44 @@ async function executeSmartLoop(ctx, params) {
           const direction = parseScrollDirection(cmd);
           if (direction) {
             globalLogger.debug(`[SmartLoop] Scrolling ${direction}`);
-            await ctx.performScroll(direction);
+            await ctx.performSwipe({ direction });
+            actionExecuted = true;
+          }
+        } else if (isSlideCommand(cmd)) {
+          const slideParams = parseSlideCommand(cmd);
+          if (slideParams) {
+            const { direction, percentage, startX, startY } = slideParams;
+            const isVertical = direction === "up" || direction === "down";
+            const distance = Math.round(
+              (isVertical ? screenshotResolution.height : screenshotResolution.width) * (percentage / 100)
+            );
+            let endX = startX;
+            let endY = startY;
+            switch (direction) {
+              case "up":
+                endY = startY + distance;
+                break;
+              case "down":
+                endY = startY - distance;
+                break;
+              case "left":
+                endX = startX - distance;
+                break;
+              case "right":
+                endX = startX + distance;
+                break;
+            }
+            globalLogger.debug(`[SmartLoop] Sliding ${direction} ${percentage}% from (${startX}, ${startY}) to (${endX}, ${endY})`);
+            await ctx.performSwipe({
+              direction,
+              x1: startX,
+              y1: startY,
+              x2: endX,
+              y2: endY,
+              screenshotWidth: screenshotResolution.width,
+              screenshotHeight: screenshotResolution.height,
+              duration: 500
+            });
             actionExecuted = true;
           }
         } else if (isTypeCommand(cmd)) {
@@ -567,7 +655,7 @@ async function executeSmartLoop(ctx, params) {
               stepNumber: params.stepNumber,
               stepDescription: params.description,
               executionData: currentExecutionData,
-              screenResolution: ctx.screenSize,
+              screenResolution: screenshotResolution,
               platform: ctx.platform,
               filepath: params.filepath
             });
@@ -840,11 +928,10 @@ class GptDriver {
     return {
       apiKey: this.apiKey,
       platform: this.appiumSessionConfig?.platform,
-      screenSize: this.appiumSessionConfig.size,
       globalActionHistory: this.globalActionHistory,
-      getScreenshot: () => this.getScreenshot(this.appiumSessionConfig),
-      performTap: (x, y) => this.performTap(x, y),
-      performScroll: (direction) => this.performScroll(direction),
+      getScreenshot: () => this.getScreenshot(this.appiumSessionConfig, false),
+      performTap: (x, y, screenshotWidth, screenshotHeight) => this.performTap(x, y, screenshotWidth, screenshotHeight),
+      performSwipe: (params) => this.performSwipe(params),
       performType: (text) => this.performType(text),
       logCodeExecution: async (screenshot, command) => this.logCodeExecution(screenshot, command),
       organisationId: this.organisationId
@@ -897,15 +984,31 @@ class GptDriver {
   /**
    * Performs a tap action at the specified coordinates.
    */
-  async performTap(x, y) {
+  async performTap(x, y, screenshotWidth, screenshotHeight) {
     const client = await this.getWdioClient();
+    const platform = this.appiumSessionConfig?.platform;
+    const { width: w, height: h } = this.appiumSessionConfig?.size ?? { width: 1080, height: 1920 };
+    const scaled = this.scaleForIOS(
+      x,
+      y,
+      platform,
+      screenshotWidth,
+      screenshotHeight,
+      w,
+      h
+    );
+    const clampedX = this.clamp(scaled.x, 0, w - 1);
+    const clampedY = this.clamp(scaled.y, 0, h - 1);
+    globalLogger.debug(
+      `[Tap] Platform: ${platform}, Input: ${x},${y}, Window: ${w}x${h}, Final: ${clampedX},${clampedY}`
+    );
     await client.performActions([
       {
         type: "pointer",
         id: "finger1",
         parameters: { pointerType: "touch" },
         actions: [
-          { type: "pointerMove", duration: 0, x, y },
+          { type: "pointerMove", duration: 0, x: clampedX, y: clampedY },
           { type: "pointerDown", button: 0 },
           { type: "pause", duration: 100 },
           { type: "pointerUp", button: 0 }
@@ -915,25 +1018,125 @@ class GptDriver {
   }
   async performType(text) {
     const client = await this.getWdioClient();
-    await client.keys(text.split(""));
+    const platform = this.appiumSessionConfig?.platform;
+    if (platform === "iOS") {
+      const actions = text.split("").flatMap((char) => [
+        { type: "keyDown", value: char },
+        { type: "keyUp", value: char }
+      ]);
+      await client.performActions([
+        {
+          type: "key",
+          id: "keyboard",
+          actions
+        }
+      ]);
+    } else {
+      await client.keys(text.split(""));
+    }
+  }
+  clamp(value, min, max) {
+    return Math.max(min, Math.min(max, value));
+  }
+  scaleForIOS(x, y, platform, screenshotWidth, screenshotHeight, windowWidth, windowHeight) {
+    if (platform !== "iOS" || !screenshotWidth || !screenshotHeight || !windowWidth || !windowHeight) {
+      return { x: Math.round(x), y: Math.round(y) };
+    }
+    const scaleX = windowWidth / screenshotWidth;
+    const scaleY = windowHeight / screenshotHeight;
+    return {
+      x: Math.round(x * scaleX),
+      y: Math.round(y * scaleY)
+    };
   }
-  async performScroll(direction) {
+  async performSwipe(params) {
     const client = await this.getWdioClient();
-    const w = this.appiumSessionConfig?.size?.width ?? 1080;
-    const h = this.appiumSessionConfig?.size?.height ?? 1920;
-    const x = Math.round(w / 2);
-    const startY = direction === "down" ? Math.round(h * 0.8) : Math.round(h * 0.2);
-    const endY = direction === "down" ? Math.round(h * 0.2) : Math.round(h * 0.8);
+    const platform = this.appiumSessionConfig?.platform;
+    const { width: w, height: h } = this.appiumSessionConfig?.size ?? { width: 1080, height: 1920 };
+    const defaultStartX = w / 2;
+    let defaultStartY;
+    if (params.direction === "down") {
+      defaultStartY = h * 0.75;
+    } else if (params.direction === "up") {
+      defaultStartY = h * 0.25;
+    } else {
+      defaultStartY = h / 2;
+    }
+    let startX;
+    let startY;
+    if (params.x1 !== void 0 || params.y1 !== void 0) {
+      const scaled = this.scaleForIOS(
+        params.x1 ?? defaultStartX,
+        params.y1 ?? defaultStartY,
+        platform,
+        params.screenshotWidth,
+        params.screenshotHeight,
+        w,
+        h
+      );
+      startX = scaled.x;
+      startY = scaled.y;
+    } else {
+      startX = Math.round(defaultStartX);
+      startY = Math.round(defaultStartY);
+    }
+    startX = this.clamp(startX, 0, w - 1);
+    startY = this.clamp(startY, 0, h - 1);
+    let endX;
+    let endY;
+    if (params.x2 !== void 0 || params.y2 !== void 0) {
+      const scaled = this.scaleForIOS(
+        params.x2 ?? startX,
+        params.y2 ?? startY,
+        platform,
+        params.screenshotWidth,
+        params.screenshotHeight,
+        w,
+        h
+      );
+      endX = scaled.x;
+      endY = scaled.y;
+    } else {
+      const deltaX = Math.round(w * 0.5);
+      const deltaY = Math.round(h * 0.5);
+      switch (params.direction) {
+        case "left":
+          endX = Math.max(0, startX - deltaX);
+          endY = startY;
+          break;
+        case "right":
+          endX = Math.min(w - 1, startX + deltaX);
+          endY = startY;
+          break;
+        case "up":
+          endX = startX;
+          endY = Math.min(h - 1, startY + deltaY);
+          break;
+        case "down":
+          endX = startX;
+          endY = Math.max(0, startY - deltaY);
+          break;
+        default:
+          endX = startX;
+          endY = startY;
+      }
+    }
+    endX = this.clamp(endX, 0, w - 1);
+    endY = this.clamp(endY, 0, h - 1);
+    globalLogger.debug(
+      `[Swipe] Platform: ${platform}, Direction: ${params.direction}, Start: ${startX},${startY}, End: ${endX},${endY}`
+    );
+    const duration = params.duration ?? 500;
     await client.performActions([
       {
         type: "pointer",
         id: "finger1",
         parameters: { pointerType: "touch" },
         actions: [
-          { type: "pointerMove", duration: 0, x, y: startY },
+          { type: "pointerMove", duration: 0, x: startX, y: startY },
           { type: "pointerDown", button: 0 },
           { type: "pause", duration: 100 },
-          { type: "pointerMove", duration: 500, x, y: endY },
+          { type: "pointerMove", duration, x: endX, y: endY },
           { type: "pointerUp", button: 0 }
         ]
       }
@@ -952,17 +1155,17 @@ class GptDriver {
       if (found) {
         return;
       }
-      await this.performScroll(direction);
+      await this.performSwipe({ direction });
       await this._delay(500);
     }
     throw new Error(`scrollUntil target not found after ${max} scroll(s)`);
   }
-  async getScreenshot(appiumSessionConfig) {
+  async getScreenshot(appiumSessionConfig, shouldScale = true) {
     globalLogger.debug("Capturing screenshot...");
     const url = buildUrl(this.appiumSessionConfig.serverUrl, `/session/${this.appiumSessionConfig.id}/screenshot`);
     const screenshotResponse = await axios.get(url);
     let screenshot = await screenshotResponse.data.value;
-    if (appiumSessionConfig.platform === "iOS") {
+    if (appiumSessionConfig.platform === "iOS" && shouldScale) {
       globalLogger.debug(`Resizing iOS screenshot to ${appiumSessionConfig.size.width}x${appiumSessionConfig.size.height}`);
       const imageBuffer = Buffer.from(screenshot, "base64");
       const transformedImage = await sharp(imageBuffer).resize(appiumSessionConfig.size.width, appiumSessionConfig.size.height).toBuffer();
@@ -1348,7 +1551,7 @@ ${issues}`);
       }
       return val.data;
     };
-    const expandSteps = async (steps, inheritedParams, parentDir, stack) => {
+    const expandSteps = async (steps, inheritedParams, parentDir, stack, currentFilePath) => {
       const out = [];
       for (const step of steps) {
         if (step.type === "fileRef") {
@@ -1362,17 +1565,23 @@ ${issues}`);
           const child = await loadFlow(refPath);
           const mergedParams = { ...inheritedParams, ...step.overrides ?? {} };
           const childDir = path.dirname(refPath);
-          const childExpanded = await expandSteps(child.steps, mergedParams, childDir, [...stack, refKey]);
+          const childRelativePath = path.relative(baseDir, refPath).replace(/^\.\//, "");
+          const childExpanded = await expandSteps(child.steps, mergedParams, childDir, [...stack, refKey], childRelativePath);
           out.push(...childExpanded);
         } else {
-          const resolved = { ...step, __params: { ...inheritedParams } };
+          const resolved = {
+            ...step,
+            __params: { ...inheritedParams },
+            __filepath: currentFilePath
+          };
           out.push(resolved);
         }
       }
       return out;
     };
     const effectiveParams = { ...rootFlow.params ?? {} };
-    const expandedSteps = await expandSteps(rootFlow.steps, effectiveParams, baseDir, [absolutePath]);
+    const rootRelativePath = path.relative(baseDir, absolutePath).replace(/^\.\//, "");
+    const expandedSteps = await expandSteps(rootFlow.steps, effectiveParams, baseDir, [absolutePath], rootRelativePath);
     if (!this.appiumSessionStarted) {
       await this.startSession();
     }
@@ -1381,7 +1590,8 @@ ${issues}`);
     try {
       for (const step of expandedSteps) {
         const params = step.__params ?? effectiveParams;
-        const prefix = `Step #${executed + 1} [${step.type}${step.optional ? ", optional" : ""}]`;
+        const filepath = step.__filepath ?? rootRelativePath;
+        const prefix = `Step #${executed + 1} [${step.type}${step.optional ? ", optional" : ""}] (filepath: ${filepath})`;
         try {
           switch (step.type) {
             case "ai": {
@@ -1392,7 +1602,8 @@ ${issues}`);
                 const result = await executeSmartLoop(ctx, {
                   stepNumber: this.step_number,
                   description: instruction,
-                  instruction
+                  instruction,
+                  filepath
                 });
                 if (!result.success) {
                   throw new Error(result.error || "Smart loop execution failed");
@@ -1414,7 +1625,8 @@ ${issues}`);
                 const result = await executeSmartLoop(ctx, {
                   stepNumber: this.step_number,
                   description,
-                  instruction: description
+                  instruction: description,
+                  filepath
                 });
                 if (!result.success) {
                   throw new Error(result.error || "Smart loop execution failed");
@@ -1437,7 +1649,8 @@ ${issues}`);
                 const result = await executeSmartLoop(ctx, {
                   stepNumber: this.step_number,
                   description,
-                  instruction
+                  instruction,
+                  filepath
                 });
                 if (!result.success) {
                   throw new Error(result.error || "Smart loop execution failed");
@@ -1456,11 +1669,39 @@ ${issues}`);
               this.step_number++;
               break;
             }
-            case "scroll": {
-              globalLogger.info(`${prefix}: Scroll ${step.direction}`);
-              await this.takeScreenshotAndLogCodeExecution(`scroll: direction=${step.direction}`);
-              await this.performScroll(step.direction);
-              this.step_number++;
+            case "scroll":
+            case "swipe": {
+              const description = step.descriptionText ? this.interpolateTemplate(step.descriptionText, params) : void 0;
+              if (description && useSmartLoop) {
+                globalLogger.info(`${prefix}: ${description}`);
+                const ctx = this.createSmartLoopContext();
+                const result = await executeSmartLoop(ctx, {
+                  stepNumber: this.step_number,
+                  description,
+                  instruction: description,
+                  filepath
+                });
+                if (!result.success) {
+                  throw new Error(result.error || "Smart loop execution failed");
+                }
+                this.step_number++;
+              } else {
+                globalLogger.info(`${prefix}: ${step.type} ${step.direction}`);
+                await this.takeScreenshotAndLogCodeExecution(`${step.type}: direction=${step.direction}`);
+                if (step.type === "swipe") {
+                  await this.performSwipe({
+                    direction: step.direction,
+                    x1: step.x1,
+                    y1: step.y1,
+                    x2: step.x2,
+                    y2: step.y2,
+                    duration: step.duration
+                  });
+                } else {
+                  await this.performSwipe({ direction: step.direction });
+                }
+                this.step_number++;
+              }
               break;
             }
             case "zoom": {

package/dist/index.d.cts

@@ -121,6 +121,40 @@ declare const SavableTestStoreSchema: z.ZodObject<{
         id: z.ZodOptional<z.ZodNumber>;
         descriptionText: z.ZodOptional<z.ZodString>;
         optional: z.ZodOptional<z.ZodBoolean>;
+    } & {
+        type: z.ZodLiteral<"swipe">;
+        direction: z.ZodEnum<["left", "right", "up", "down"]>;
+        x1: z.ZodOptional<z.ZodNumber>;
+        y1: z.ZodOptional<z.ZodNumber>;
+        x2: z.ZodOptional<z.ZodNumber>;
+        y2: z.ZodOptional<z.ZodNumber>;
+        duration: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+    }, "strip", z.ZodTypeAny, {
+        type: "swipe";
+        direction: "left" | "right" | "up" | "down";
+        duration: number;
+        id?: number | undefined;
+        descriptionText?: string | undefined;
+        optional?: boolean | undefined;
+        x1?: number | undefined;
+        y1?: number | undefined;
+        x2?: number | undefined;
+        y2?: number | undefined;
+    }, {
+        type: "swipe";
+        direction: "left" | "right" | "up" | "down";
+        id?: number | undefined;
+        descriptionText?: string | undefined;
+        optional?: boolean | undefined;
+        x1?: number | undefined;
+        y1?: number | undefined;
+        x2?: number | undefined;
+        y2?: number | undefined;
+        duration?: number | undefined;
+    }>, z.ZodObject<{
+        id: z.ZodOptional<z.ZodNumber>;
+        descriptionText: z.ZodOptional<z.ZodString>;
+        optional: z.ZodOptional<z.ZodBoolean>;
     } & {
         type: z.ZodLiteral<"scroll">;
         direction: z.ZodEnum<["up", "down"]>;
@@ -271,6 +305,17 @@ declare const SavableTestStoreSchema: z.ZodObject<{
         id?: number | undefined;
         descriptionText?: string | undefined;
         optional?: boolean | undefined;
+    } | {
+        type: "swipe";
+        direction: "left" | "right" | "up" | "down";
+        duration: number;
+        id?: number | undefined;
+        descriptionText?: string | undefined;
+        optional?: boolean | undefined;
+        x1?: number | undefined;
+        y1?: number | undefined;
+        x2?: number | undefined;
+        y2?: number | undefined;
     } | {
         type: "scroll";
         direction: "up" | "down";
@@ -339,6 +384,17 @@ declare const SavableTestStoreSchema: z.ZodObject<{
         id?: number | undefined;
         descriptionText?: string | undefined;
         optional?: boolean | undefined;
+    } | {
+        type: "swipe";
+        direction: "left" | "right" | "up" | "down";
+        id?: number | undefined;
+        descriptionText?: string | undefined;
+        optional?: boolean | undefined;
+        x1?: number | undefined;
+        y1?: number | undefined;
+        x2?: number | undefined;
+        y2?: number | undefined;
+        duration?: number | undefined;
     } | {
         type: "scroll";
         direction: "up" | "down";
@@ -461,7 +517,9 @@ declare class GptDriver {
      */
     private performTap;
     private performType;
-    private performScroll;
+    private clamp;
+    private scaleForIOS;
+    private performSwipe;
     private getPageSource;
     private performScrollUntil;
     private getScreenshot;

package/dist/index.mjs

@@ -20,6 +20,15 @@ function buildUrl(base, extraPath) {
   }
   return `${baseUrl}${extraPath}`;
 }
+const getImageDimensions = async (base64) => {
+  const base64Data = base64.replace(/^data:image\/\w+;base64,/, "");
+  const buffer = Buffer.from(base64Data, "base64");
+  const metadata = await sharp(buffer).metadata();
+  if (!metadata.width || !metadata.height) {
+    throw new Error("Unable to get image dimensions");
+  }
+  return { width: metadata.width, height: metadata.height };
+};
 
 const colors = {
   reset: "\x1B[0m",
@@ -105,6 +114,15 @@ const SavableTypeStepSchema = SavableStepBaseSchema.extend({
   type: z.literal("type"),
   text: z.string()
 });
+const SavableSwipeStepSchema = SavableStepBaseSchema.extend({
+  type: z.literal("swipe"),
+  direction: z.enum(["left", "right", "up", "down"]),
+  x1: z.number().optional(),
+  y1: z.number().optional(),
+  x2: z.number().optional(),
+  y2: z.number().optional(),
+  duration: z.number().optional().default(500)
+});
 const SavableScrollStepSchema = SavableStepBaseSchema.extend({
   type: z.literal("scroll"),
   direction: z.enum(["up", "down"])
@@ -140,6 +158,8 @@ const SavableStepSchema = z.discriminatedUnion("type", [
   // type: 'assert'
   SavableTypeStepSchema,
   // type: 'type'
+  SavableSwipeStepSchema,
+  // type: 'swipe'
   SavableScrollStepSchema,
   // type: 'scroll'
   SavableZoomStepSchema,
@@ -425,6 +445,30 @@ function isScrollCommand(cmd) {
 function isTypeCommand(cmd) {
   return cmd.startsWith("type:");
 }
+function isSlideCommand(cmd) {
+  return cmd.startsWith("slide");
+}
+function parseSlideCommand(cmd) {
+  const slideMatch = cmd.match(
+    /slide\s+(up|down|left|right)\s+(\d+)%(?::\s*[^;]*)?;(\d+);(\d+)/i
+  );
+  if (!slideMatch) {
+    return null;
+  }
+  const extractedDirection = slideMatch[1].toLowerCase();
+  const directionMap = {
+    down: "up",
+    up: "down",
+    left: "right",
+    right: "left"
+  };
+  return {
+    direction: directionMap[extractedDirection],
+    percentage: parseInt(slideMatch[2], 10),
+    startX: parseInt(slideMatch[3], 10),
+    startY: parseInt(slideMatch[4], 10)
+  };
+}
 
 async function executeSmartLoop(ctx, params) {
   const maxCacheAttempts = Math.floor(CACHE_RETRY_MS / CACHE_CHECK_INTERVAL_MS);
@@ -440,6 +484,8 @@ async function executeSmartLoop(ctx, params) {
       let screenshot = "";
       let commands = [];
       let isCacheHit = false;
+      const firstScreenshot = await ctx.getScreenshot();
+      const screenshotResolution = await getImageDimensions(firstScreenshot);
       for (let attempt = 0; attempt < maxCacheAttempts; attempt++) {
         screenshot = await ctx.getScreenshot();
         const sizeInBytes = screenshot.length * 0.75;
@@ -452,7 +498,7 @@ async function executeSmartLoop(ctx, params) {
             stepNumber: params.stepNumber,
             stepDescription: params.description,
             screenshot,
-            screenResolution: ctx.screenSize,
+            screenResolution: screenshotResolution,
             highestUsedIndex: lastCacheIndex,
             platform: ctx.platform,
             filepath: params.filepath
@@ -523,7 +569,12 @@ async function executeSmartLoop(ctx, params) {
           const coords = parseTapCoordinates(cmd);
           if (coords) {
             globalLogger.debug(`[SmartLoop] Executing tap at (${coords.x}, ${coords.y})`);
-            await ctx.performTap(coords.x, coords.y);
+            await ctx.performTap(
+              coords.x,
+              coords.y,
+              screenshotResolution.width,
+              screenshotResolution.height
+            );
             actionExecuted = true;
           }
         } else if (isWaitCommand(cmd)) {
@@ -537,7 +588,44 @@ async function executeSmartLoop(ctx, params) {
           const direction = parseScrollDirection(cmd);
           if (direction) {
             globalLogger.debug(`[SmartLoop] Scrolling ${direction}`);
-            await ctx.performScroll(direction);
+            await ctx.performSwipe({ direction });
+            actionExecuted = true;
+          }
+        } else if (isSlideCommand(cmd)) {
+          const slideParams = parseSlideCommand(cmd);
+          if (slideParams) {
+            const { direction, percentage, startX, startY } = slideParams;
+            const isVertical = direction === "up" || direction === "down";
+            const distance = Math.round(
+              (isVertical ? screenshotResolution.height : screenshotResolution.width) * (percentage / 100)
+            );
+            let endX = startX;
+            let endY = startY;
+            switch (direction) {
+              case "up":
+                endY = startY + distance;
+                break;
+              case "down":
+                endY = startY - distance;
+                break;
+              case "left":
+                endX = startX - distance;
+                break;
+              case "right":
+                endX = startX + distance;
+                break;
+            }
+            globalLogger.debug(`[SmartLoop] Sliding ${direction} ${percentage}% from (${startX}, ${startY}) to (${endX}, ${endY})`);
+            await ctx.performSwipe({
+              direction,
+              x1: startX,
+              y1: startY,
+              x2: endX,
+              y2: endY,
+              screenshotWidth: screenshotResolution.width,
+              screenshotHeight: screenshotResolution.height,
+              duration: 500
+            });
             actionExecuted = true;
           }
         } else if (isTypeCommand(cmd)) {
@@ -565,7 +653,7 @@ async function executeSmartLoop(ctx, params) {
               stepNumber: params.stepNumber,
               stepDescription: params.description,
               executionData: currentExecutionData,
-              screenResolution: ctx.screenSize,
+              screenResolution: screenshotResolution,
               platform: ctx.platform,
               filepath: params.filepath
             });
@@ -838,11 +926,10 @@ class GptDriver {
     return {
       apiKey: this.apiKey,
       platform: this.appiumSessionConfig?.platform,
-      screenSize: this.appiumSessionConfig.size,
       globalActionHistory: this.globalActionHistory,
-      getScreenshot: () => this.getScreenshot(this.appiumSessionConfig),
-      performTap: (x, y) => this.performTap(x, y),
-      performScroll: (direction) => this.performScroll(direction),
+      getScreenshot: () => this.getScreenshot(this.appiumSessionConfig, false),
+      performTap: (x, y, screenshotWidth, screenshotHeight) => this.performTap(x, y, screenshotWidth, screenshotHeight),
+      performSwipe: (params) => this.performSwipe(params),
       performType: (text) => this.performType(text),
       logCodeExecution: async (screenshot, command) => this.logCodeExecution(screenshot, command),
       organisationId: this.organisationId
@@ -895,15 +982,31 @@ class GptDriver {
   /**
    * Performs a tap action at the specified coordinates.
    */
-  async performTap(x, y) {
+  async performTap(x, y, screenshotWidth, screenshotHeight) {
     const client = await this.getWdioClient();
+    const platform = this.appiumSessionConfig?.platform;
+    const { width: w, height: h } = this.appiumSessionConfig?.size ?? { width: 1080, height: 1920 };
+    const scaled = this.scaleForIOS(
+      x,
+      y,
+      platform,
+      screenshotWidth,
+      screenshotHeight,
+      w,
+      h
+    );
+    const clampedX = this.clamp(scaled.x, 0, w - 1);
+    const clampedY = this.clamp(scaled.y, 0, h - 1);
+    globalLogger.debug(
+      `[Tap] Platform: ${platform}, Input: ${x},${y}, Window: ${w}x${h}, Final: ${clampedX},${clampedY}`
+    );
     await client.performActions([
       {
         type: "pointer",
         id: "finger1",
         parameters: { pointerType: "touch" },
         actions: [
-          { type: "pointerMove", duration: 0, x, y },
+          { type: "pointerMove", duration: 0, x: clampedX, y: clampedY },
           { type: "pointerDown", button: 0 },
           { type: "pause", duration: 100 },
           { type: "pointerUp", button: 0 }
@@ -913,25 +1016,125 @@ class GptDriver {
   }
   async performType(text) {
     const client = await this.getWdioClient();
-    await client.keys(text.split(""));
+    const platform = this.appiumSessionConfig?.platform;
+    if (platform === "iOS") {
+      const actions = text.split("").flatMap((char) => [
+        { type: "keyDown", value: char },
+        { type: "keyUp", value: char }
+      ]);
+      await client.performActions([
+        {
+          type: "key",
+          id: "keyboard",
+          actions
+        }
+      ]);
+    } else {
+      await client.keys(text.split(""));
+    }
+  }
+  clamp(value, min, max) {
+    return Math.max(min, Math.min(max, value));
+  }
+  scaleForIOS(x, y, platform, screenshotWidth, screenshotHeight, windowWidth, windowHeight) {
+    if (platform !== "iOS" || !screenshotWidth || !screenshotHeight || !windowWidth || !windowHeight) {
+      return { x: Math.round(x), y: Math.round(y) };
+    }
+    const scaleX = windowWidth / screenshotWidth;
+    const scaleY = windowHeight / screenshotHeight;
+    return {
+      x: Math.round(x * scaleX),
+      y: Math.round(y * scaleY)
+    };
   }
-  async performScroll(direction) {
+  async performSwipe(params) {
     const client = await this.getWdioClient();
-    const w = this.appiumSessionConfig?.size?.width ?? 1080;
-    const h = this.appiumSessionConfig?.size?.height ?? 1920;
-    const x = Math.round(w / 2);
-    const startY = direction === "down" ? Math.round(h * 0.8) : Math.round(h * 0.2);
-    const endY = direction === "down" ? Math.round(h * 0.2) : Math.round(h * 0.8);
+    const platform = this.appiumSessionConfig?.platform;
+    const { width: w, height: h } = this.appiumSessionConfig?.size ?? { width: 1080, height: 1920 };
+    const defaultStartX = w / 2;
+    let defaultStartY;
+    if (params.direction === "down") {
+      defaultStartY = h * 0.75;
+    } else if (params.direction === "up") {
+      defaultStartY = h * 0.25;
+    } else {
+      defaultStartY = h / 2;
+    }
+    let startX;
+    let startY;
+    if (params.x1 !== void 0 || params.y1 !== void 0) {
+      const scaled = this.scaleForIOS(
+        params.x1 ?? defaultStartX,
+        params.y1 ?? defaultStartY,
+        platform,
+        params.screenshotWidth,
+        params.screenshotHeight,
+        w,
+        h
+      );
+      startX = scaled.x;
+      startY = scaled.y;
+    } else {
+      startX = Math.round(defaultStartX);
+      startY = Math.round(defaultStartY);
+    }
+    startX = this.clamp(startX, 0, w - 1);
+    startY = this.clamp(startY, 0, h - 1);
+    let endX;
+    let endY;
+    if (params.x2 !== void 0 || params.y2 !== void 0) {
+      const scaled = this.scaleForIOS(
+        params.x2 ?? startX,
+        params.y2 ?? startY,
+        platform,
+        params.screenshotWidth,
+        params.screenshotHeight,
+        w,
+        h
+      );
+      endX = scaled.x;
+      endY = scaled.y;
+    } else {
+      const deltaX = Math.round(w * 0.5);
+      const deltaY = Math.round(h * 0.5);
+      switch (params.direction) {
+        case "left":
+          endX = Math.max(0, startX - deltaX);
+          endY = startY;
+          break;
+        case "right":
+          endX = Math.min(w - 1, startX + deltaX);
+          endY = startY;
+          break;
+        case "up":
+          endX = startX;
+          endY = Math.min(h - 1, startY + deltaY);
+          break;
+        case "down":
+          endX = startX;
+          endY = Math.max(0, startY - deltaY);
+          break;
+        default:
+          endX = startX;
+          endY = startY;
+      }
+    }
+    endX = this.clamp(endX, 0, w - 1);
+    endY = this.clamp(endY, 0, h - 1);
+    globalLogger.debug(
+      `[Swipe] Platform: ${platform}, Direction: ${params.direction}, Start: ${startX},${startY}, End: ${endX},${endY}`
+    );
+    const duration = params.duration ?? 500;
     await client.performActions([
       {
         type: "pointer",
         id: "finger1",
         parameters: { pointerType: "touch" },
         actions: [
-          { type: "pointerMove", duration: 0, x, y: startY },
+          { type: "pointerMove", duration: 0, x: startX, y: startY },
           { type: "pointerDown", button: 0 },
           { type: "pause", duration: 100 },
-          { type: "pointerMove", duration: 500, x, y: endY },
+          { type: "pointerMove", duration, x: endX, y: endY },
           { type: "pointerUp", button: 0 }
         ]
       }
@@ -950,17 +1153,17 @@ class GptDriver {
       if (found) {
         return;
       }
-      await this.performScroll(direction);
+      await this.performSwipe({ direction });
       await this._delay(500);
     }
     throw new Error(`scrollUntil target not found after ${max} scroll(s)`);
   }
-  async getScreenshot(appiumSessionConfig) {
+  async getScreenshot(appiumSessionConfig, shouldScale = true) {
     globalLogger.debug("Capturing screenshot...");
     const url = buildUrl(this.appiumSessionConfig.serverUrl, `/session/${this.appiumSessionConfig.id}/screenshot`);
     const screenshotResponse = await axios.get(url);
     let screenshot = await screenshotResponse.data.value;
-    if (appiumSessionConfig.platform === "iOS") {
+    if (appiumSessionConfig.platform === "iOS" && shouldScale) {
       globalLogger.debug(`Resizing iOS screenshot to ${appiumSessionConfig.size.width}x${appiumSessionConfig.size.height}`);
       const imageBuffer = Buffer.from(screenshot, "base64");
       const transformedImage = await sharp(imageBuffer).resize(appiumSessionConfig.size.width, appiumSessionConfig.size.height).toBuffer();
@@ -1346,7 +1549,7 @@ ${issues}`);
       }
       return val.data;
     };
-    const expandSteps = async (steps, inheritedParams, parentDir, stack) => {
+    const expandSteps = async (steps, inheritedParams, parentDir, stack, currentFilePath) => {
       const out = [];
       for (const step of steps) {
         if (step.type === "fileRef") {
@@ -1360,17 +1563,23 @@ ${issues}`);
           const child = await loadFlow(refPath);
           const mergedParams = { ...inheritedParams, ...step.overrides ?? {} };
           const childDir = path.dirname(refPath);
-          const childExpanded = await expandSteps(child.steps, mergedParams, childDir, [...stack, refKey]);
+          const childRelativePath = path.relative(baseDir, refPath).replace(/^\.\//, "");
+          const childExpanded = await expandSteps(child.steps, mergedParams, childDir, [...stack, refKey], childRelativePath);
           out.push(...childExpanded);
         } else {
-          const resolved = { ...step, __params: { ...inheritedParams } };
+          const resolved = {
+            ...step,
+            __params: { ...inheritedParams },
+            __filepath: currentFilePath
+          };
           out.push(resolved);
         }
       }
       return out;
     };
     const effectiveParams = { ...rootFlow.params ?? {} };
-    const expandedSteps = await expandSteps(rootFlow.steps, effectiveParams, baseDir, [absolutePath]);
+    const rootRelativePath = path.relative(baseDir, absolutePath).replace(/^\.\//, "");
+    const expandedSteps = await expandSteps(rootFlow.steps, effectiveParams, baseDir, [absolutePath], rootRelativePath);
     if (!this.appiumSessionStarted) {
       await this.startSession();
     }
@@ -1379,7 +1588,8 @@ ${issues}`);
     try {
       for (const step of expandedSteps) {
         const params = step.__params ?? effectiveParams;
-        const prefix = `Step #${executed + 1} [${step.type}${step.optional ? ", optional" : ""}]`;
+        const filepath = step.__filepath ?? rootRelativePath;
+        const prefix = `Step #${executed + 1} [${step.type}${step.optional ? ", optional" : ""}] (filepath: ${filepath})`;
         try {
           switch (step.type) {
             case "ai": {
@@ -1390,7 +1600,8 @@ ${issues}`);
                 const result = await executeSmartLoop(ctx, {
                   stepNumber: this.step_number,
                   description: instruction,
-                  instruction
+                  instruction,
+                  filepath
                 });
                 if (!result.success) {
                   throw new Error(result.error || "Smart loop execution failed");
@@ -1412,7 +1623,8 @@ ${issues}`);
                 const result = await executeSmartLoop(ctx, {
                   stepNumber: this.step_number,
                   description,
-                  instruction: description
+                  instruction: description,
+                  filepath
                 });
                 if (!result.success) {
                   throw new Error(result.error || "Smart loop execution failed");
@@ -1435,7 +1647,8 @@ ${issues}`);
                 const result = await executeSmartLoop(ctx, {
                   stepNumber: this.step_number,
                   description,
-                  instruction
+                  instruction,
+                  filepath
                 });
                 if (!result.success) {
                   throw new Error(result.error || "Smart loop execution failed");
@@ -1454,11 +1667,39 @@ ${issues}`);
               this.step_number++;
               break;
             }
-            case "scroll": {
-              globalLogger.info(`${prefix}: Scroll ${step.direction}`);
-              await this.takeScreenshotAndLogCodeExecution(`scroll: direction=${step.direction}`);
-              await this.performScroll(step.direction);
-              this.step_number++;
+            case "scroll":
+            case "swipe": {
+              const description = step.descriptionText ? this.interpolateTemplate(step.descriptionText, params) : void 0;
+              if (description && useSmartLoop) {
+                globalLogger.info(`${prefix}: ${description}`);
+                const ctx = this.createSmartLoopContext();
+                const result = await executeSmartLoop(ctx, {
+                  stepNumber: this.step_number,
+                  description,
+                  instruction: description,
+                  filepath
+                });
+                if (!result.success) {
+                  throw new Error(result.error || "Smart loop execution failed");
+                }
+                this.step_number++;
+              } else {
+                globalLogger.info(`${prefix}: ${step.type} ${step.direction}`);
+                await this.takeScreenshotAndLogCodeExecution(`${step.type}: direction=${step.direction}`);
+                if (step.type === "swipe") {
+                  await this.performSwipe({
+                    direction: step.direction,
+                    x1: step.x1,
+                    y1: step.y1,
+                    x2: step.x2,
+                    y2: step.y2,
+                    duration: step.duration
+                  });
+                } else {
+                  await this.performSwipe({ direction: step.direction });
+                }
+                this.step_number++;
+              }
               break;
             }
             case "zoom": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpt-driver-node",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "main": "./dist/index.cjs",
   "module": "./dist/index.mjs",
   "types": "./dist/index.d.cts",