gpt-driver-node 1.0.9 → 1.0.11

package/dist/index.cjs

@@ -684,6 +684,7 @@ async function executeSmartLoop(ctx, params) {
         screenshotResolution = await getImageDimensions(screenshot2);
       }
       let aiCommands = [];
+      let reasoning = [];
       let actionDescription = [];
       if (!isCacheHit) {
         anyCacheMiss = true;
@@ -727,7 +728,9 @@ async function executeSmartLoop(ctx, params) {
           actionDescription = gptCommands.slice(actionDescriptionIndex, actionDescriptionIndex + 1);
         }
         const reasoningIndex = gptCommands.findIndex((entry) => entry.startsWith("reasoning:"));
+        reasoning = [];
         if (reasoningIndex !== -1) {
+          reasoning = gptCommands.at(reasoningIndex).split("reasoning:").at(1).trim().split("- ");
           const parsedCommands = gptCommands.slice(reasoningIndex);
           const rememberCommands = parsedCommands.filter((cmd) => isRememberCommand(cmd));
           if (rememberCommands.length > 0) {
@@ -742,7 +745,13 @@ async function executeSmartLoop(ctx, params) {
         screenshot,
         commands
       });
-      const logPromise = ctx.logCodeExecution(screenshot, commands.join("\n"), isCacheHit);
+      const logPromise = ctx.logAIExecution({
+        screenshot,
+        prompt: params.instruction,
+        commands: aiCommands,
+        reasoning,
+        fromCache: isCacheHit
+      });
       if (ctx.pendingLogPromises) {
         ctx.pendingLogPromises.push(logPromise);
       }
@@ -1113,7 +1122,10 @@ class GptDriver {
         device_config: {
           platform: this.appiumSessionConfig?.platform ?? this.gptDriverCloudConfig?.platform,
           device: this.appiumSessionConfig?.deviceName ?? this.gptDriverCloudConfig?.deviceName,
-          os: this.appiumSessionConfig?.platformVersion ?? this.gptDriverCloudConfig?.platformVersion
+          os: this.appiumSessionConfig?.platformVersion ?? this.gptDriverCloudConfig?.platformVersion,
+          ...this.appiumSessionConfig?.size && {
+            screenResolution: `${this.appiumSessionConfig.size.width}x${this.appiumSessionConfig.size.height}`
+          }
         },
         use_internal_virtual_device: this.useGptDriverCloud,
         build_id: this.buildId,
@@ -1236,7 +1248,7 @@ ${"=".repeat(50)}`);
       performSwipe: (params) => this.performSwipe(params),
       performType: (text) => this.performType(text),
       performPressEnter: () => this.performPressEnter(),
-      logCodeExecution: async (screenshot, command, isCacheHit) => this.logCodeExecution(screenshot, command, isCacheHit),
+      logAIExecution: async (params) => this.logAIExecution(params),
       organisationId: this.organisationId,
       middleLayerAssertFn: options?.middleLayerAssertFn,
       pendingLogPromises: options?.pendingLogPromises
@@ -2194,7 +2206,13 @@ ${issues}`);
             case "type": {
               const text = this.interpolateTemplate(step.text, params);
               globalLogger.info(`${prefix}: Type text`);
-              this.takeScreenshotAndLogCodeExecution(`type: text=${text}`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `type: text=${text}`,
+                  commands: ["Local type execution"],
+                  reasoning: ["Local type execution"]
+                })
+              );
               await this.performType(text);
               if (isFromFileRef) {
                 fileRefStepNumber++;
@@ -2205,7 +2223,13 @@ ${issues}`);
             }
             case "enter": {
               globalLogger.info(`${prefix}: Press Enter`);
-              this.takeScreenshotAndLogCodeExecution(`press: Enter`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `press: Enter`,
+                  commands: ["Local press Enter execution"],
+                  reasoning: ["Local press Enter execution"]
+                })
+              );
               await this.performPressEnter();
               if (step.delayNextStep) {
                 await this._delay(step.delayNextStep);
@@ -2242,7 +2266,13 @@ ${issues}`);
                 }
               } else {
                 globalLogger.info(`${prefix}: ${step.type} ${step.direction}`);
-                this.takeScreenshotAndLogCodeExecution(`${step.type}: direction=${step.direction}`);
+                this.pendingLogPromises.push(
+                  this.takeScreenshotAndLogAIExecution({
+                    prompt: `${step.type}: direction=${step.direction}`,
+                    commands: [`Local ${step.type} execution`],
+                    reasoning: [`Local ${step.type} execution`]
+                  })
+                );
                 await this.performSwipe({
                   direction: step.direction,
                   x1: step.x1,
@@ -2263,7 +2293,13 @@ ${issues}`);
             }
             case "zoom": {
               globalLogger.info(`${prefix}: Zoom ${step.direction}`);
-              this.takeScreenshotAndLogCodeExecution(`zoom: direction=${step.direction}`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `zoom: direction=${step.direction}`,
+                  commands: [`Local zoom execution`],
+                  reasoning: [`Local zoom execution`]
+                })
+              );
               if (isFromFileRef) {
                 fileRefStepNumber++;
               } else {
@@ -2274,7 +2310,13 @@ ${issues}`);
             case "scrollUntil": {
               const interpolatedText = step.text != null ? this.interpolateTemplate(step.text, params) : void 0;
               globalLogger.info(`${prefix}: Scroll until ${interpolatedText ?? step.elementId}`);
-              this.takeScreenshotAndLogCodeExecution(`scrollUntil: text=${interpolatedText}, elementId=${step.elementId}`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `scrollUntil: text=${interpolatedText}, elementId=${step.elementId}`,
+                  commands: [`Local scrollUntil execution`],
+                  reasoning: [`Local scrollUntil execution`]
+                })
+              );
               await this.performScrollUntil({
                 direction: step.direction,
                 text: interpolatedText,
@@ -2293,7 +2335,13 @@ ${issues}`);
               const bundleId = params["bundleId"];
               const url = this.interpolateTemplate(step.url, params);
               globalLogger.info(`${prefix}: Open deeplink ${url}`);
-              this.takeScreenshotAndLogCodeExecution(`openDeepLinkUrl: url=${url}`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `openDeepLinkUrl: url=${url}`,
+                  commands: [`Local openDeepLinkUrl execution`],
+                  reasoning: [`Local openDeepLinkUrl execution`]
+                })
+              );
               await this.openDeepLinkUrl({ url, package: pkg, bundleId });
               break;
             }
@@ -2414,6 +2462,28 @@ ${"=".repeat(50)}`);
       globalLogger.error("Failed to log code execution", e);
     }
   }
+  async logAIExecution(params) {
+    try {
+      await axios.post(`${this.gptDriverBaseUrl}/sessions/${this.gptDriverSessionId}/log_ai_execution`, {
+        api_key: this.apiKey,
+        base64_screenshot: params.screenshot,
+        prompt: params.prompt,
+        reasoning: params.reasoning,
+        commands: params.commands,
+        from_cache: !!params.fromCache
+      });
+    } catch (e) {
+      globalLogger.error("Failed to log code execution", e);
+    }
+  }
+  async takeScreenshotAndLogAIExecution(params) {
+    try {
+      const screenshot = await this.getScreenshot(this.appiumSessionConfig);
+      await this.logAIExecution({ ...params, screenshot });
+    } catch (e) {
+      globalLogger.error("Failed to log code execution", e);
+    }
+  }
   async takeScreenshotAndLogCodeExecution(command) {
     try {
       const screenshot = await this.getScreenshot(this.appiumSessionConfig);

package/dist/index.d.cts

@@ -877,6 +877,8 @@ declare class GptDriver {
     private gptHandler;
     private executeCommand;
     private logCodeExecution;
+    private logAIExecution;
+    private takeScreenshotAndLogAIExecution;
     private takeScreenshotAndLogCodeExecution;
 }
 

package/dist/index.mjs

@@ -682,6 +682,7 @@ async function executeSmartLoop(ctx, params) {
         screenshotResolution = await getImageDimensions(screenshot2);
       }
       let aiCommands = [];
+      let reasoning = [];
       let actionDescription = [];
       if (!isCacheHit) {
         anyCacheMiss = true;
@@ -725,7 +726,9 @@ async function executeSmartLoop(ctx, params) {
           actionDescription = gptCommands.slice(actionDescriptionIndex, actionDescriptionIndex + 1);
         }
         const reasoningIndex = gptCommands.findIndex((entry) => entry.startsWith("reasoning:"));
+        reasoning = [];
         if (reasoningIndex !== -1) {
+          reasoning = gptCommands.at(reasoningIndex).split("reasoning:").at(1).trim().split("- ");
           const parsedCommands = gptCommands.slice(reasoningIndex);
           const rememberCommands = parsedCommands.filter((cmd) => isRememberCommand(cmd));
           if (rememberCommands.length > 0) {
@@ -740,7 +743,13 @@ async function executeSmartLoop(ctx, params) {
         screenshot,
         commands
       });
-      const logPromise = ctx.logCodeExecution(screenshot, commands.join("\n"), isCacheHit);
+      const logPromise = ctx.logAIExecution({
+        screenshot,
+        prompt: params.instruction,
+        commands: aiCommands,
+        reasoning,
+        fromCache: isCacheHit
+      });
       if (ctx.pendingLogPromises) {
         ctx.pendingLogPromises.push(logPromise);
       }
@@ -1111,7 +1120,10 @@ class GptDriver {
         device_config: {
           platform: this.appiumSessionConfig?.platform ?? this.gptDriverCloudConfig?.platform,
           device: this.appiumSessionConfig?.deviceName ?? this.gptDriverCloudConfig?.deviceName,
-          os: this.appiumSessionConfig?.platformVersion ?? this.gptDriverCloudConfig?.platformVersion
+          os: this.appiumSessionConfig?.platformVersion ?? this.gptDriverCloudConfig?.platformVersion,
+          ...this.appiumSessionConfig?.size && {
+            screenResolution: `${this.appiumSessionConfig.size.width}x${this.appiumSessionConfig.size.height}`
+          }
         },
         use_internal_virtual_device: this.useGptDriverCloud,
         build_id: this.buildId,
@@ -1234,7 +1246,7 @@ ${"=".repeat(50)}`);
       performSwipe: (params) => this.performSwipe(params),
       performType: (text) => this.performType(text),
       performPressEnter: () => this.performPressEnter(),
-      logCodeExecution: async (screenshot, command, isCacheHit) => this.logCodeExecution(screenshot, command, isCacheHit),
+      logAIExecution: async (params) => this.logAIExecution(params),
       organisationId: this.organisationId,
       middleLayerAssertFn: options?.middleLayerAssertFn,
       pendingLogPromises: options?.pendingLogPromises
@@ -2192,7 +2204,13 @@ ${issues}`);
             case "type": {
               const text = this.interpolateTemplate(step.text, params);
               globalLogger.info(`${prefix}: Type text`);
-              this.takeScreenshotAndLogCodeExecution(`type: text=${text}`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `type: text=${text}`,
+                  commands: ["Local type execution"],
+                  reasoning: ["Local type execution"]
+                })
+              );
               await this.performType(text);
               if (isFromFileRef) {
                 fileRefStepNumber++;
@@ -2203,7 +2221,13 @@ ${issues}`);
             }
             case "enter": {
               globalLogger.info(`${prefix}: Press Enter`);
-              this.takeScreenshotAndLogCodeExecution(`press: Enter`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `press: Enter`,
+                  commands: ["Local press Enter execution"],
+                  reasoning: ["Local press Enter execution"]
+                })
+              );
               await this.performPressEnter();
               if (step.delayNextStep) {
                 await this._delay(step.delayNextStep);
@@ -2240,7 +2264,13 @@ ${issues}`);
                 }
               } else {
                 globalLogger.info(`${prefix}: ${step.type} ${step.direction}`);
-                this.takeScreenshotAndLogCodeExecution(`${step.type}: direction=${step.direction}`);
+                this.pendingLogPromises.push(
+                  this.takeScreenshotAndLogAIExecution({
+                    prompt: `${step.type}: direction=${step.direction}`,
+                    commands: [`Local ${step.type} execution`],
+                    reasoning: [`Local ${step.type} execution`]
+                  })
+                );
                 await this.performSwipe({
                   direction: step.direction,
                   x1: step.x1,
@@ -2261,7 +2291,13 @@ ${issues}`);
             }
             case "zoom": {
               globalLogger.info(`${prefix}: Zoom ${step.direction}`);
-              this.takeScreenshotAndLogCodeExecution(`zoom: direction=${step.direction}`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `zoom: direction=${step.direction}`,
+                  commands: [`Local zoom execution`],
+                  reasoning: [`Local zoom execution`]
+                })
+              );
               if (isFromFileRef) {
                 fileRefStepNumber++;
               } else {
@@ -2272,7 +2308,13 @@ ${issues}`);
             case "scrollUntil": {
               const interpolatedText = step.text != null ? this.interpolateTemplate(step.text, params) : void 0;
               globalLogger.info(`${prefix}: Scroll until ${interpolatedText ?? step.elementId}`);
-              this.takeScreenshotAndLogCodeExecution(`scrollUntil: text=${interpolatedText}, elementId=${step.elementId}`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `scrollUntil: text=${interpolatedText}, elementId=${step.elementId}`,
+                  commands: [`Local scrollUntil execution`],
+                  reasoning: [`Local scrollUntil execution`]
+                })
+              );
               await this.performScrollUntil({
                 direction: step.direction,
                 text: interpolatedText,
@@ -2291,7 +2333,13 @@ ${issues}`);
               const bundleId = params["bundleId"];
               const url = this.interpolateTemplate(step.url, params);
               globalLogger.info(`${prefix}: Open deeplink ${url}`);
-              this.takeScreenshotAndLogCodeExecution(`openDeepLinkUrl: url=${url}`);
+              this.pendingLogPromises.push(
+                this.takeScreenshotAndLogAIExecution({
+                  prompt: `openDeepLinkUrl: url=${url}`,
+                  commands: [`Local openDeepLinkUrl execution`],
+                  reasoning: [`Local openDeepLinkUrl execution`]
+                })
+              );
               await this.openDeepLinkUrl({ url, package: pkg, bundleId });
               break;
             }
@@ -2412,6 +2460,28 @@ ${"=".repeat(50)}`);
       globalLogger.error("Failed to log code execution", e);
     }
   }
+  async logAIExecution(params) {
+    try {
+      await axios.post(`${this.gptDriverBaseUrl}/sessions/${this.gptDriverSessionId}/log_ai_execution`, {
+        api_key: this.apiKey,
+        base64_screenshot: params.screenshot,
+        prompt: params.prompt,
+        reasoning: params.reasoning,
+        commands: params.commands,
+        from_cache: !!params.fromCache
+      });
+    } catch (e) {
+      globalLogger.error("Failed to log code execution", e);
+    }
+  }
+  async takeScreenshotAndLogAIExecution(params) {
+    try {
+      const screenshot = await this.getScreenshot(this.appiumSessionConfig);
+      await this.logAIExecution({ ...params, screenshot });
+    } catch (e) {
+      globalLogger.error("Failed to log code execution", e);
+    }
+  }
   async takeScreenshotAndLogCodeExecution(command) {
     try {
       const screenshot = await this.getScreenshot(this.appiumSessionConfig);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpt-driver-node",
-  "version": "1.0.9",
+  "version": "1.0.11",
   "main": "./dist/index.cjs",
   "module": "./dist/index.mjs",
   "types": "./dist/index.d.cts",