gpt-driver-node 1.0.10 → 1.0.12

package/CHANGELOG.md

@@ -0,0 +1,65 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+See [docs.mobileboost.io](https://docs.mobileboost.io) for full documentation.
+
+## [1.0.12] - 2026-02-27
+
+### Added
+- `maxWaitForStableScreenSecs` config option to control the maximum time to wait for a stable screen
+
+### Changed
+- Enhanced screen stability handling for more reliable test execution
+
+## [1.0.11] - 2026-02-25
+
+### Changed
+- Improved AI execution logging with richer detail (screenshot, prompt, commands, reasoning, cache status)
+
+## [1.0.10] - 2026-02-25
+
+### Added
+- Screen resolution is now automatically sent to the server during session initialization when a device `size` is provided
+
+## [1.0.9] - 2026-02-20
+
+### Changed
+- Smart loop now throws an error on failure instead of resolving silently — update your error handling accordingly
+
+## [1.0.8] - 2026-02-19
+
+### Changed
+- Screenshots are now compressed before being sent, reducing bandwidth usage
+- ADB-based screenshots are used automatically on Android when the Appium server is local
+
+## [1.0.7] - 2026-02-18
+
+### Added
+- `additionalUserContext` option in `GptDriverConfig` — pass extra context to the AI for more accurate test execution
+
+### Fixed
+- Session is now correctly marked as failed when `maxCacheAttempts` is reached
+
+## [1.0.6] - 2026-02-09
+
+### Added
+- Automatic retry logic for middleware service calls
+- ADB screenshot support for Android devices
+- `usemiddleLayerAssertFn` option for assertion handling
+
+### Changed
+- Screen stability checks are now performed once per loop iteration for better cache efficiency
+- Default log level changed to `info`
+
+## [1.0.5] - 2026-01-12
+
+### Added
+- Press/Enter key command support
+- Config file and environment variable support via `configFilePath`
+- `actionDescription` field on commands for improved traceability
+- Screen stability wait between steps to increase cache hit rate
+
+## [1.0.3] - 2026-01-02
+
+### Changed
+- `executeFlow` reliability and performance improvements

package/dist/index.cjs

@@ -632,7 +632,7 @@ async function executeSmartLoop(ctx, params) {
       let commands = [];
       let isCacheHit = false;
       let screenshotResolution = void 0;
-      const stabilityResult = await waitForStableScreen(ctx.getScreenshot);
+      const stabilityResult = await waitForStableScreen(ctx.getScreenshot, { ...ctx.maxWaitForStableScreenSecs && { maxTimeoutSec: ctx.maxWaitForStableScreenSecs } });
       for (let attempt = 0; attempt < maxCacheAttempts; attempt++) {
         const screenshotStartTime = performance.now();
         const screenshotEndTime = performance.now();
@@ -684,6 +684,7 @@ async function executeSmartLoop(ctx, params) {
         screenshotResolution = await getImageDimensions(screenshot2);
       }
       let aiCommands = [];
+      let reasoning = [];
       let actionDescription = [];
       if (!isCacheHit) {
         anyCacheMiss = true;
@@ -727,7 +728,9 @@ async function executeSmartLoop(ctx, params) {
           actionDescription = gptCommands.slice(actionDescriptionIndex, actionDescriptionIndex + 1);
         }
         const reasoningIndex = gptCommands.findIndex((entry) => entry.startsWith("reasoning:"));
+        reasoning = [];
         if (reasoningIndex !== -1) {
+          reasoning = gptCommands.at(reasoningIndex).split("reasoning:").at(1).trim().split("- ");
           const parsedCommands = gptCommands.slice(reasoningIndex);
           const rememberCommands = parsedCommands.filter((cmd) => isRememberCommand(cmd));
           if (rememberCommands.length > 0) {
@@ -742,7 +745,13 @@ async function executeSmartLoop(ctx, params) {
         screenshot,
         commands
       });
-      const logPromise = ctx.logCodeExecution(screenshot, commands.join("\n"), isCacheHit);
+      const logPromise = ctx.logAIExecution({
+        screenshot,
+        prompt: params.instruction,
+        commands: aiCommands,
+        reasoning,
+        fromCache: isCacheHit
+      });
       if (ctx.pendingLogPromises) {
         ctx.pendingLogPromises.push(logPromise);
       }
@@ -955,6 +964,7 @@ class GptDriver {
   organisationId;
   configFilePath;
   additionalUserContext;
+  maxWaitForStableScreenSecs;
   // Session Execution Stats
   _stats_startTime = 0;
   _stats_executedSteps = 0;
@@ -991,6 +1001,7 @@ class GptDriver {
     this.organisationId = config.organisationId;
     this.configFilePath = config.configFilePath;
     this.additionalUserContext = config.additionalUserContext;
+    this.maxWaitForStableScreenSecs = config.maxWaitForStableScreenSecs;
     if (config.useGptDriverCloud) {
       if (config.serverConfig.device?.platform == null) {
         throw new Error("Platform is missing. Please specify the platform when using GPTDriver Cloud.");
@@ -1239,10 +1250,11 @@ ${"=".repeat(50)}`);
       performSwipe: (params) => this.performSwipe(params),
       performType: (text) => this.performType(text),
       performPressEnter: () => this.performPressEnter(),
-      logCodeExecution: async (screenshot, command, isCacheHit) => this.logCodeExecution(screenshot, command, isCacheHit),
+      logAIExecution: async (params) => this.logAIExecution(params),
       organisationId: this.organisationId,
       middleLayerAssertFn: options?.middleLayerAssertFn,
-      pendingLogPromises: options?.pendingLogPromises
+      pendingLogPromises: options?.pendingLogPromises,
+      maxWaitForStableScreenSecs: this.maxWaitForStableScreenSecs
     };
   }
   /**
@@ -1318,7 +1330,7 @@ ${"=".repeat(50)}`);
         actions: [
           { type: "pointerMove", duration: 0, x: clampedX, y: clampedY },
           { type: "pointerDown", button: 0 },
-          { type: "pause", duration: 100 },
+          { type: "pause", duration: 300 },
           { type: "pointerUp", button: 0 }
         ]
       }
@@ -1571,7 +1583,7 @@ ${"=".repeat(50)}`);
     const driver = this.driver;
     if (appiumHandler != null) {
       try {
-        await this.takeScreenshotAndLogCodeExecution(appiumHandler.toString());
+        this.pendingLogPromises.push(this.takeScreenshotAndLogCodeExecution(appiumHandler.toString()));
         await appiumHandler(driver);
         globalLogger.debug("Custom Appium handler executed successfully");
       } catch (e) {
@@ -1603,7 +1615,7 @@ ${"=".repeat(50)}`);
       try {
         const handlerStartTime = performance.now();
         globalLogger.debug(`[Performance] Executing custom Appium handler...`);
-        await this.takeScreenshotAndLogCodeExecution(appiumHandler.toString());
+        this.pendingLogPromises.push(this.takeScreenshotAndLogCodeExecution(appiumHandler.toString()));
         await appiumHandler(driver);
         const handlerEndTime = performance.now();
         globalLogger.debug("Custom Appium handler executed successfully");
@@ -2197,7 +2209,13 @@ ${issues}`);
             case "type": {
               const text = this.interpolateTemplate(step.text, params);
               globalLogger.info(`${prefix}: Type text`);
-              this.takeScreenshotAndLogCodeExecution(`type: text=${text}`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `type: text=${text}`,
+                  commands: ["Local type execution"],
+                  reasoning: ["Local type execution"]
+                })
+              );
               await this.performType(text);
               if (isFromFileRef) {
                 fileRefStepNumber++;
@@ -2208,7 +2226,13 @@ ${issues}`);
             }
             case "enter": {
               globalLogger.info(`${prefix}: Press Enter`);
-              this.takeScreenshotAndLogCodeExecution(`press: Enter`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `press: Enter`,
+                  commands: ["Local press Enter execution"],
+                  reasoning: ["Local press Enter execution"]
+                })
+              );
               await this.performPressEnter();
               if (step.delayNextStep) {
                 await this._delay(step.delayNextStep);
@@ -2245,7 +2269,13 @@ ${issues}`);
                 }
               } else {
                 globalLogger.info(`${prefix}: ${step.type} ${step.direction}`);
-                this.takeScreenshotAndLogCodeExecution(`${step.type}: direction=${step.direction}`);
+                this.pendingLogPromises.push(
+                  this.takeScreenshotAndLogAIExecution({
+                    prompt: `${step.type}: direction=${step.direction}`,
+                    commands: [`Local ${step.type} execution`],
+                    reasoning: [`Local ${step.type} execution`]
+                  })
+                );
                 await this.performSwipe({
                   direction: step.direction,
                   x1: step.x1,
@@ -2266,7 +2296,13 @@ ${issues}`);
             }
             case "zoom": {
               globalLogger.info(`${prefix}: Zoom ${step.direction}`);
-              this.takeScreenshotAndLogCodeExecution(`zoom: direction=${step.direction}`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `zoom: direction=${step.direction}`,
+                  commands: [`Local zoom execution`],
+                  reasoning: [`Local zoom execution`]
+                })
+              );
               if (isFromFileRef) {
                 fileRefStepNumber++;
               } else {
@@ -2277,7 +2313,13 @@ ${issues}`);
             case "scrollUntil": {
               const interpolatedText = step.text != null ? this.interpolateTemplate(step.text, params) : void 0;
               globalLogger.info(`${prefix}: Scroll until ${interpolatedText ?? step.elementId}`);
-              this.takeScreenshotAndLogCodeExecution(`scrollUntil: text=${interpolatedText}, elementId=${step.elementId}`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `scrollUntil: text=${interpolatedText}, elementId=${step.elementId}`,
+                  commands: [`Local scrollUntil execution`],
+                  reasoning: [`Local scrollUntil execution`]
+                })
+              );
               await this.performScrollUntil({
                 direction: step.direction,
                 text: interpolatedText,
@@ -2296,7 +2338,13 @@ ${issues}`);
               const bundleId = params["bundleId"];
               const url = this.interpolateTemplate(step.url, params);
               globalLogger.info(`${prefix}: Open deeplink ${url}`);
-              this.takeScreenshotAndLogCodeExecution(`openDeepLinkUrl: url=${url}`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `openDeepLinkUrl: url=${url}`,
+                  commands: [`Local openDeepLinkUrl execution`],
+                  reasoning: [`Local openDeepLinkUrl execution`]
+                })
+              );
               await this.openDeepLinkUrl({ url, package: pkg, bundleId });
               break;
             }
@@ -2417,6 +2465,28 @@ ${"=".repeat(50)}`);
       globalLogger.error("Failed to log code execution", e);
     }
   }
+  async logAIExecution(params) {
+    try {
+      await axios.post(`${this.gptDriverBaseUrl}/sessions/${this.gptDriverSessionId}/log_ai_execution`, {
+        api_key: this.apiKey,
+        base64_screenshot: params.screenshot,
+        prompt: params.prompt,
+        reasoning: params.reasoning,
+        commands: params.commands,
+        from_cache: !!params.fromCache
+      });
+    } catch (e) {
+      globalLogger.error("Failed to log code execution", e);
+    }
+  }
+  async takeScreenshotAndLogAIExecution(params) {
+    try {
+      const screenshot = await this.getScreenshot(this.appiumSessionConfig);
+      await this.logAIExecution({ ...params, screenshot });
+    } catch (e) {
+      globalLogger.error("Failed to log code execution", e);
+    }
+  }
   async takeScreenshotAndLogCodeExecution(command) {
     try {
       const screenshot = await this.getScreenshot(this.appiumSessionConfig);

package/dist/index.d.cts

@@ -25,6 +25,7 @@ interface GptDriverConfig {
     additionalUserContext?: string;
     organisationId?: string;
     configFilePath?: string;
+    maxWaitForStableScreenSecs?: number;
 }
 /**
  * Parameters for opening a deep link url in the Appium session.
@@ -514,6 +515,7 @@ declare class GptDriver {
     private organisationId?;
     private configFilePath?;
     private additionalUserContext?;
+    private maxWaitForStableScreenSecs?;
     private _stats_startTime;
     private _stats_executedSteps;
     private _stats_cacheHits;
@@ -877,6 +879,8 @@ declare class GptDriver {
     private gptHandler;
     private executeCommand;
     private logCodeExecution;
+    private logAIExecution;
+    private takeScreenshotAndLogAIExecution;
     private takeScreenshotAndLogCodeExecution;
 }
 

package/dist/index.mjs

@@ -630,7 +630,7 @@ async function executeSmartLoop(ctx, params) {
       let commands = [];
       let isCacheHit = false;
       let screenshotResolution = void 0;
-      const stabilityResult = await waitForStableScreen(ctx.getScreenshot);
+      const stabilityResult = await waitForStableScreen(ctx.getScreenshot, { ...ctx.maxWaitForStableScreenSecs && { maxTimeoutSec: ctx.maxWaitForStableScreenSecs } });
       for (let attempt = 0; attempt < maxCacheAttempts; attempt++) {
         const screenshotStartTime = performance.now();
         const screenshotEndTime = performance.now();
@@ -682,6 +682,7 @@ async function executeSmartLoop(ctx, params) {
         screenshotResolution = await getImageDimensions(screenshot2);
       }
       let aiCommands = [];
+      let reasoning = [];
       let actionDescription = [];
       if (!isCacheHit) {
         anyCacheMiss = true;
@@ -725,7 +726,9 @@ async function executeSmartLoop(ctx, params) {
           actionDescription = gptCommands.slice(actionDescriptionIndex, actionDescriptionIndex + 1);
         }
         const reasoningIndex = gptCommands.findIndex((entry) => entry.startsWith("reasoning:"));
+        reasoning = [];
         if (reasoningIndex !== -1) {
+          reasoning = gptCommands.at(reasoningIndex).split("reasoning:").at(1).trim().split("- ");
           const parsedCommands = gptCommands.slice(reasoningIndex);
           const rememberCommands = parsedCommands.filter((cmd) => isRememberCommand(cmd));
           if (rememberCommands.length > 0) {
@@ -740,7 +743,13 @@ async function executeSmartLoop(ctx, params) {
         screenshot,
         commands
       });
-      const logPromise = ctx.logCodeExecution(screenshot, commands.join("\n"), isCacheHit);
+      const logPromise = ctx.logAIExecution({
+        screenshot,
+        prompt: params.instruction,
+        commands: aiCommands,
+        reasoning,
+        fromCache: isCacheHit
+      });
       if (ctx.pendingLogPromises) {
         ctx.pendingLogPromises.push(logPromise);
       }
@@ -953,6 +962,7 @@ class GptDriver {
   organisationId;
   configFilePath;
   additionalUserContext;
+  maxWaitForStableScreenSecs;
   // Session Execution Stats
   _stats_startTime = 0;
   _stats_executedSteps = 0;
@@ -989,6 +999,7 @@ class GptDriver {
     this.organisationId = config.organisationId;
     this.configFilePath = config.configFilePath;
     this.additionalUserContext = config.additionalUserContext;
+    this.maxWaitForStableScreenSecs = config.maxWaitForStableScreenSecs;
     if (config.useGptDriverCloud) {
       if (config.serverConfig.device?.platform == null) {
         throw new Error("Platform is missing. Please specify the platform when using GPTDriver Cloud.");
@@ -1237,10 +1248,11 @@ ${"=".repeat(50)}`);
       performSwipe: (params) => this.performSwipe(params),
       performType: (text) => this.performType(text),
       performPressEnter: () => this.performPressEnter(),
-      logCodeExecution: async (screenshot, command, isCacheHit) => this.logCodeExecution(screenshot, command, isCacheHit),
+      logAIExecution: async (params) => this.logAIExecution(params),
       organisationId: this.organisationId,
       middleLayerAssertFn: options?.middleLayerAssertFn,
-      pendingLogPromises: options?.pendingLogPromises
+      pendingLogPromises: options?.pendingLogPromises,
+      maxWaitForStableScreenSecs: this.maxWaitForStableScreenSecs
     };
   }
   /**
@@ -1316,7 +1328,7 @@ ${"=".repeat(50)}`);
         actions: [
           { type: "pointerMove", duration: 0, x: clampedX, y: clampedY },
           { type: "pointerDown", button: 0 },
-          { type: "pause", duration: 100 },
+          { type: "pause", duration: 300 },
           { type: "pointerUp", button: 0 }
         ]
       }
@@ -1569,7 +1581,7 @@ ${"=".repeat(50)}`);
     const driver = this.driver;
     if (appiumHandler != null) {
       try {
-        await this.takeScreenshotAndLogCodeExecution(appiumHandler.toString());
+        this.pendingLogPromises.push(this.takeScreenshotAndLogCodeExecution(appiumHandler.toString()));
         await appiumHandler(driver);
         globalLogger.debug("Custom Appium handler executed successfully");
       } catch (e) {
@@ -1601,7 +1613,7 @@ ${"=".repeat(50)}`);
       try {
         const handlerStartTime = performance.now();
         globalLogger.debug(`[Performance] Executing custom Appium handler...`);
-        await this.takeScreenshotAndLogCodeExecution(appiumHandler.toString());
+        this.pendingLogPromises.push(this.takeScreenshotAndLogCodeExecution(appiumHandler.toString()));
         await appiumHandler(driver);
         const handlerEndTime = performance.now();
         globalLogger.debug("Custom Appium handler executed successfully");
@@ -2195,7 +2207,13 @@ ${issues}`);
             case "type": {
               const text = this.interpolateTemplate(step.text, params);
               globalLogger.info(`${prefix}: Type text`);
-              this.takeScreenshotAndLogCodeExecution(`type: text=${text}`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `type: text=${text}`,
+                  commands: ["Local type execution"],
+                  reasoning: ["Local type execution"]
+                })
+              );
               await this.performType(text);
               if (isFromFileRef) {
                 fileRefStepNumber++;
@@ -2206,7 +2224,13 @@ ${issues}`);
             }
             case "enter": {
               globalLogger.info(`${prefix}: Press Enter`);
-              this.takeScreenshotAndLogCodeExecution(`press: Enter`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `press: Enter`,
+                  commands: ["Local press Enter execution"],
+                  reasoning: ["Local press Enter execution"]
+                })
+              );
               await this.performPressEnter();
               if (step.delayNextStep) {
                 await this._delay(step.delayNextStep);
@@ -2243,7 +2267,13 @@ ${issues}`);
                 }
               } else {
                 globalLogger.info(`${prefix}: ${step.type} ${step.direction}`);
-                this.takeScreenshotAndLogCodeExecution(`${step.type}: direction=${step.direction}`);
+                this.pendingLogPromises.push(
+                  this.takeScreenshotAndLogAIExecution({
+                    prompt: `${step.type}: direction=${step.direction}`,
+                    commands: [`Local ${step.type} execution`],
+                    reasoning: [`Local ${step.type} execution`]
+                  })
+                );
                 await this.performSwipe({
                   direction: step.direction,
                   x1: step.x1,
@@ -2264,7 +2294,13 @@ ${issues}`);
             }
             case "zoom": {
               globalLogger.info(`${prefix}: Zoom ${step.direction}`);
-              this.takeScreenshotAndLogCodeExecution(`zoom: direction=${step.direction}`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `zoom: direction=${step.direction}`,
+                  commands: [`Local zoom execution`],
+                  reasoning: [`Local zoom execution`]
+                })
+              );
               if (isFromFileRef) {
                 fileRefStepNumber++;
               } else {
@@ -2275,7 +2311,13 @@ ${issues}`);
             case "scrollUntil": {
               const interpolatedText = step.text != null ? this.interpolateTemplate(step.text, params) : void 0;
               globalLogger.info(`${prefix}: Scroll until ${interpolatedText ?? step.elementId}`);
-              this.takeScreenshotAndLogCodeExecution(`scrollUntil: text=${interpolatedText}, elementId=${step.elementId}`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `scrollUntil: text=${interpolatedText}, elementId=${step.elementId}`,
+                  commands: [`Local scrollUntil execution`],
+                  reasoning: [`Local scrollUntil execution`]
+                })
+              );
               await this.performScrollUntil({
                 direction: step.direction,
                 text: interpolatedText,
@@ -2294,7 +2336,13 @@ ${issues}`);
               const bundleId = params["bundleId"];
               const url = this.interpolateTemplate(step.url, params);
               globalLogger.info(`${prefix}: Open deeplink ${url}`);
-              this.takeScreenshotAndLogCodeExecution(`openDeepLinkUrl: url=${url}`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `openDeepLinkUrl: url=${url}`,
+                  commands: [`Local openDeepLinkUrl execution`],
+                  reasoning: [`Local openDeepLinkUrl execution`]
+                })
+              );
               await this.openDeepLinkUrl({ url, package: pkg, bundleId });
               break;
             }
@@ -2415,6 +2463,28 @@ ${"=".repeat(50)}`);
       globalLogger.error("Failed to log code execution", e);
     }
   }
+  async logAIExecution(params) {
+    try {
+      await axios.post(`${this.gptDriverBaseUrl}/sessions/${this.gptDriverSessionId}/log_ai_execution`, {
+        api_key: this.apiKey,
+        base64_screenshot: params.screenshot,
+        prompt: params.prompt,
+        reasoning: params.reasoning,
+        commands: params.commands,
+        from_cache: !!params.fromCache
+      });
+    } catch (e) {
+      globalLogger.error("Failed to log code execution", e);
+    }
+  }
+  async takeScreenshotAndLogAIExecution(params) {
+    try {
+      const screenshot = await this.getScreenshot(this.appiumSessionConfig);
+      await this.logAIExecution({ ...params, screenshot });
+    } catch (e) {
+      globalLogger.error("Failed to log code execution", e);
+    }
+  }
   async takeScreenshotAndLogCodeExecution(command) {
     try {
       const screenshot = await this.getScreenshot(this.appiumSessionConfig);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpt-driver-node",
-  "version": "1.0.10",
+  "version": "1.0.12",
   "main": "./dist/index.cjs",
   "module": "./dist/index.mjs",
   "types": "./dist/index.d.cts",