misoai-web 1.5.6 → 1.5.8

package/dist/lib/midscene-playground.js

@@ -41,7 +41,8 @@ var WebElementInfo = class {
     id,
     attributes,
     indexId,
-    xpaths
+    xpaths,
+    isVisible
   }) {
     this.content = content;
     this.rect = rect;
@@ -54,6 +55,7 @@ var WebElementInfo = class {
     this.attributes = attributes;
     this.indexId = indexId;
     this.xpaths = xpaths;
+    this.isVisible = isVisible;
   }
 };
 
@@ -76,14 +78,15 @@ async function parseContextFromWebPage(page, _opt) {
     })
   ]);
   const webTree = (0, import_extractor.traverseTree)(tree, (elementInfo) => {
-    const { rect, id, content, attributes, locator, indexId } = elementInfo;
+    const { rect, id, content, attributes, locator, indexId, isVisible } = elementInfo;
     return new WebElementInfo({
       rect,
       locator,
       id,
       content,
       attributes,
-      indexId
+      indexId,
+      isVisible
     });
   });
   (0, import_utils2.assert)(screenshotBase64, "screenshotBase64 is required");
@@ -114,7 +117,7 @@ function printReportMsg(filepath) {
 }
 var ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED = "NOT_IMPLEMENTED_AS_DESIGNED";
 function replaceIllegalPathCharsAndSpace(str) {
-  return str.replace(/[/\\:*?"<>| ]/g, "-");
+  return str.replace(/[:*?"<>| ]/g, "-");
 }
 function matchElementFromPlan(planLocateParam, tree) {
   if (!planLocateParam) {
@@ -244,6 +247,10 @@ var ScriptPlayer = class {
       } else if ("aiQuery" in flowItem) {
         const queryTask = flowItem;
         const prompt = queryTask.aiQuery;
+        const options = {
+          domIncluded: queryTask.domIncluded,
+          screenshotIncluded: queryTask.screenshotIncluded
+        };
         (0, import_utils3.assert)(prompt, "missing prompt for aiQuery");
         (0, import_utils3.assert)(
           typeof prompt === "string",
@@ -254,6 +261,10 @@ var ScriptPlayer = class {
       } else if ("aiNumber" in flowItem) {
         const numberTask = flowItem;
         const prompt = numberTask.aiNumber;
+        const options = {
+          domIncluded: numberTask.domIncluded,
+          screenshotIncluded: numberTask.screenshotIncluded
+        };
         (0, import_utils3.assert)(prompt, "missing prompt for number");
         (0, import_utils3.assert)(
           typeof prompt === "string",
@@ -264,6 +275,10 @@ var ScriptPlayer = class {
       } else if ("aiString" in flowItem) {
         const stringTask = flowItem;
         const prompt = stringTask.aiString;
+        const options = {
+          domIncluded: stringTask.domIncluded,
+          screenshotIncluded: stringTask.screenshotIncluded
+        };
         (0, import_utils3.assert)(prompt, "missing prompt for string");
         (0, import_utils3.assert)(
           typeof prompt === "string",
@@ -274,6 +289,10 @@ var ScriptPlayer = class {
       } else if ("aiBoolean" in flowItem) {
         const booleanTask = flowItem;
         const prompt = booleanTask.aiBoolean;
+        const options = {
+          domIncluded: booleanTask.domIncluded,
+          screenshotIncluded: booleanTask.screenshotIncluded
+        };
         (0, import_utils3.assert)(prompt, "missing prompt for boolean");
         (0, import_utils3.assert)(
           typeof prompt === "string",
@@ -316,6 +335,9 @@ var ScriptPlayer = class {
       } else if ("aiTap" in flowItem) {
         const tapTask = flowItem;
         await agent.aiTap(tapTask.aiTap, tapTask);
+      } else if ("aiRightClick" in flowItem) {
+        const rightClickTask = flowItem;
+        await agent.aiRightClick(rightClickTask.aiRightClick, rightClickTask);
       } else if ("aiHover" in flowItem) {
         const hoverTask = flowItem;
         await agent.aiHover(hoverTask.aiHover, hoverTask);
@@ -338,6 +360,11 @@ var ScriptPlayer = class {
           evaluateJavaScriptTask.javascript
         );
         this.setResult(evaluateJavaScriptTask.name, result);
+      } else if ("logScreenshot" in flowItem) {
+        const logScreenshotTask = flowItem;
+        await agent.logScreenshot(logScreenshotTask.logScreenshot, {
+          content: logScreenshotTask.content || ""
+        });
       } else {
         throw new Error(`unknown flowItem: ${JSON.stringify(flowItem)}`);
       }
@@ -573,6 +600,94 @@ var replanningCountLimit = 10;
 var isAndroidPage = (page) => {
   return page.pageType === "android";
 };
+var WorkflowMemory = class {
+  constructor(config) {
+    this.workflows = /* @__PURE__ */ new Map();
+    this.config = config;
+  }
+  /**
+   * İş akışı hafızasını getirir
+   */
+  getWorkflowMemory(workflowId = "default") {
+    const workflow = this.workflows.get(workflowId);
+    return workflow?.memory || [];
+  }
+  /**
+   * İş akışı verilerini getirir
+   */
+  getWorkflowData(workflowId = "default") {
+    return this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
+  }
+  /**
+   * İş akışı hafızasını kaydeder
+   */
+  saveWorkflowMemory(memory, workflowId = "default") {
+    const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
+    workflow.memory = [...memory];
+    workflow.metadata.totalSteps = workflow.steps.length;
+    workflow.metadata.completedSteps = workflow.steps.filter((s) => s.status === "completed").length;
+    workflow.metadata.failedSteps = workflow.steps.filter((s) => s.status === "failed").length;
+    this.workflows.set(workflowId, workflow);
+    this.enforceRetentionPolicy();
+  }
+  /**
+   * İş akışı bağlamını günceller
+   */
+  updateWorkflowContext(context, workflowId = "default") {
+    const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
+    workflow.context = { ...workflow.context, ...context };
+    if (context.currentStep) {
+      const existingStep = workflow.steps.find((s) => s.stepName === context.currentStep);
+      if (!existingStep) {
+        workflow.steps.push({
+          stepId: `step_${workflow.steps.length + 1}`,
+          stepName: context.currentStep,
+          timestamp: context.timestamp,
+          status: "running",
+          memoryItems: []
+        });
+      }
+    }
+    this.workflows.set(workflowId, workflow);
+  }
+  /**
+   * İş akışını temizler
+   */
+  clearWorkflow(workflowId = "default") {
+    this.workflows.delete(workflowId);
+  }
+  /**
+   * Tüm iş akışlarını temizler
+   */
+  clearAll() {
+    this.workflows.clear();
+  }
+  createEmptyWorkflowData(workflowId) {
+    return {
+      workflowId,
+      steps: [],
+      memory: [],
+      context: {
+        pageInfo: { url: "", title: "" },
+        timestamp: Date.now()
+      },
+      metadata: {
+        totalSteps: 0,
+        completedSteps: 0,
+        failedSteps: 0,
+        startTime: Date.now()
+      }
+    };
+  }
+  enforceRetentionPolicy() {
+    const maxWorkflows = 10;
+    if (this.workflows.size > maxWorkflows) {
+      const sortedWorkflows = Array.from(this.workflows.entries()).sort(([, a], [, b]) => (b.metadata.endTime || b.metadata.startTime) - (a.metadata.endTime || a.metadata.startTime));
+      const toDelete = sortedWorkflows.slice(maxWorkflows);
+      toDelete.forEach(([workflowId]) => this.workflows.delete(workflowId));
+    }
+  }
+};
 var PageTaskExecutor = class {
   constructor(page, insight, opts) {
     this.conversationHistory = [];
@@ -580,6 +695,25 @@ var PageTaskExecutor = class {
     this.insight = insight;
     this.taskCache = opts.taskCache;
     this.onTaskStartCallback = opts?.onTaskStart;
+    this.memoryConfig = {
+      maxItems: 100,
+      maxAge: 2 * 60 * 60 * 1e3,
+      // 2 saat
+      enablePersistence: true,
+      enableAnalytics: true,
+      filterStrategy: "hybrid",
+      ...opts?.memoryConfig
+    };
+    this.sessionContext = {
+      sessionId: opts?.sessionId || this.generateSessionId(),
+      workflowId: opts?.workflowId,
+      startTime: Date.now(),
+      pageInfo: {
+        url: "",
+        title: ""
+      }
+    };
+    this.workflowMemory = new WorkflowMemory(this.memoryConfig);
   }
   async recordScreenshot(timing) {
     const base64 = await this.page.screenshotBase64();
@@ -792,8 +926,11 @@ var PageTaskExecutor = class {
               insightDump = dump;
             };
             this.insight.onceDumpUpdatedFn = dumpCollector;
+            const memoryContext = this.getMemoryAsContext();
             const assertion = await this.insight.assert(
-              assertPlan.param.assertion
+              assertPlan.param.assertion,
+              memoryContext
+              // Hafıza bağlamını geç
             );
             if (!assertion.pass) {
               if (plan2.type === "Assert") {
@@ -830,10 +967,10 @@ var PageTaskExecutor = class {
               if (!taskParam || !taskParam.value) {
                 return;
               }
-              await this.page.keyboard.type(taskParam.value);
-            } else {
-              await this.page.keyboard.type(taskParam.value);
             }
+            await this.page.keyboard.type(taskParam.value, {
+              autoDismissKeyboard: taskParam.autoDismissKeyboard
+            });
           }
         };
         tasks.push(taskActionInput);
@@ -862,6 +999,22 @@ var PageTaskExecutor = class {
           }
         };
         tasks.push(taskActionTap);
+      } else if (plan2.type === "RightClick") {
+        const taskActionRightClick = {
+          type: "Action",
+          subType: "RightClick",
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (param, { element }) => {
+            (0, import_utils6.assert)(element, "Element not found, cannot right click");
+            await this.page.mouse.click(
+              element.center[0],
+              element.center[1],
+              { button: "right" }
+            );
+          }
+        };
+        tasks.push(taskActionRightClick);
       } else if (plan2.type === "Drag") {
         const taskActionDrag = {
           type: "Action",
@@ -1263,25 +1416,146 @@ var PageTaskExecutor = class {
     };
     return task;
   }
+  /**
+   * Persistent executor'ı getirir veya oluşturur
+   */
+  getPersistentExecutor() {
+    if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
+      const previousMemory = this.workflowMemory.getWorkflowMemory(this.sessionContext.workflowId);
+      this.persistentExecutor = new import_misoai_core.Executor("Persistent Task Executor", {
+        onTaskStart: this.onTaskStartCallback,
+        initialMemory: previousMemory
+      });
+    }
+    return this.persistentExecutor;
+  }
+  /**
+   * Sayfa bağlamını günceller
+   */
+  async updatePageContext() {
+    try {
+      if (this.page.url) {
+        this.sessionContext.pageInfo.url = await this.page.url();
+      }
+      if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
+        this.sessionContext.pageInfo.title = await this.page.title();
+      }
+    } catch (e) {
+    }
+  }
+  /**
+   * Hafızayı temizler
+   */
+  clearMemory() {
+    if (this.persistentExecutor) {
+      this.persistentExecutor.clearMemory();
+    }
+    this.workflowMemory.clearWorkflow(this.sessionContext.workflowId || "default");
+  }
+  /**
+   * Mevcut hafızayı döndürür
+   */
+  getMemory() {
+    return this.persistentExecutor?.getMemory() || [];
+  }
+  /**
+   * İş akışı hafızasını döndürür
+   */
+  getWorkflowMemory() {
+    return this.workflowMemory.getWorkflowData(this.sessionContext.workflowId || "default");
+  }
+  /**
+   * Hafıza istatistiklerini döndürür
+   */
+  getMemoryStats() {
+    return this.persistentExecutor?.getMemoryStats() || {
+      totalItems: 0,
+      analytics: { totalTasks: 0, memoryHits: 0, memoryMisses: 0, averageMemorySize: 0, memoryEffectiveness: 0 },
+      config: this.memoryConfig
+    };
+  }
+  /**
+   * Hafıza konfigürasyonunu günceller
+   */
+  updateMemoryConfig(config) {
+    this.memoryConfig = { ...this.memoryConfig, ...config };
+    if (this.persistentExecutor) {
+      this.persistentExecutor.updateMemoryConfig(this.memoryConfig);
+    }
+  }
+  /**
+   * Oturum ID'sini oluşturur
+   */
+  generateSessionId() {
+    return `session_${Date.now()}_${Math.random().toString(36).substr(2, 8)}`;
+  }
+  /**
+   * Hafızayı bağlam olarak formatlar
+   */
+  getMemoryAsContext() {
+    const memory = this.getMemory();
+    if (memory.length === 0) {
+      return "";
+    }
+    const recentMemory = memory.slice(-5);
+    return recentMemory.map((item) => `- ${item.summary}`).join("\n");
+  }
   async runPlans(title, plans, opts) {
-    const taskExecutor = new import_misoai_core.Executor(title, {
-      onTaskStart: this.onTaskStartCallback
-    });
+    await this.updatePageContext();
+    const useMemory = opts?.useMemory !== false;
+    let taskExecutor;
+    if (useMemory) {
+      taskExecutor = this.getPersistentExecutor();
+      this.workflowMemory.updateWorkflowContext({
+        currentStep: title,
+        pageInfo: this.sessionContext.pageInfo,
+        timestamp: Date.now()
+      }, this.sessionContext.workflowId || "default");
+    } else {
+      taskExecutor = new import_misoai_core.Executor(title, {
+        onTaskStart: this.onTaskStartCallback
+      });
+    }
     const { tasks } = await this.convertPlanToExecutable(plans, opts);
+    tasks.forEach((task) => {
+      task.context = {
+        ...task.context,
+        ...this.sessionContext.pageInfo,
+        workflowId: this.sessionContext.workflowId,
+        sessionId: this.sessionContext.sessionId
+      };
+    });
     await taskExecutor.append(tasks);
     const result = await taskExecutor.flush();
+    if (useMemory) {
+      this.workflowMemory.saveWorkflowMemory(
+        taskExecutor.getMemory(),
+        this.sessionContext.workflowId || "default"
+      );
+    }
     return {
       output: result,
       executor: taskExecutor
     };
   }
   async action(userPrompt, actionContext, opts) {
-    const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userPrompt), {
-      onTaskStart: this.onTaskStartCallback
-    });
-    let planningTask = this.planningTaskFromPrompt(userPrompt, void 0, actionContext);
+    const useMemory = true;
+    let taskExecutor;
+    if (useMemory) {
+      taskExecutor = this.getPersistentExecutor();
+    } else {
+      taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userPrompt), {
+        onTaskStart: this.onTaskStartCallback
+      });
+    }
+    const memoryContext = this.getMemoryAsContext();
+    const initialLog = memoryContext ? memoryContext : void 0;
+    let planningTask = this.planningTaskFromPrompt(userPrompt, initialLog, actionContext);
     let replanCount = 0;
     const logList = [];
+    if (memoryContext) {
+      logList.push(memoryContext);
+    }
     const yamlFlow = [];
     while (planningTask) {
       if (replanCount > replanningCountLimit) {
@@ -1390,16 +1664,22 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async createTypeQueryTask(type, demand) {
-    const taskExecutor = new import_misoai_core.Executor(
-      taskTitleStr(
-        type,
-        typeof demand === "string" ? demand : JSON.stringify(demand)
-      ),
-      {
-        onTaskStart: this.onTaskStartCallback
-      }
-    );
+  async createTypeQueryTask(type, demand, opt) {
+    const useMemory = true;
+    let taskExecutor;
+    if (useMemory) {
+      taskExecutor = this.getPersistentExecutor();
+    } else {
+      taskExecutor = new import_misoai_core.Executor(
+        taskTitleStr(
+          type,
+          typeof demand === "string" ? demand : JSON.stringify(demand)
+        ),
+        {
+          onTaskStart: this.onTaskStartCallback
+        }
+      );
+    }
     const queryTask = {
       type: "Insight",
       subType: type,
@@ -1421,7 +1701,13 @@ var PageTaskExecutor = class {
             result: `${type}, ${demand}`
           };
         }
-        const { data, usage } = await this.insight.extract(demandInput);
+        const memoryContext = this.getMemoryAsContext();
+        const { data, usage } = await this.insight.extract(
+          demandInput,
+          opt,
+          memoryContext
+          // Hafıza bağlamını geç
+        );
         let outputResult = data;
         if (ifTypeRestricted) {
           (0, import_utils6.assert)(data?.result !== void 0, "No result in query data");
@@ -1441,23 +1727,29 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async query(demand) {
-    return this.createTypeQueryTask("Query", demand);
+  async query(demand, opt) {
+    return this.createTypeQueryTask("Query", demand, opt);
   }
-  async boolean(prompt) {
-    return this.createTypeQueryTask("Boolean", prompt);
+  async boolean(prompt, opt) {
+    return this.createTypeQueryTask("Boolean", prompt, opt);
   }
-  async number(prompt) {
-    return this.createTypeQueryTask("Number", prompt);
+  async number(prompt, opt) {
+    return this.createTypeQueryTask("Number", prompt, opt);
   }
-  async string(prompt) {
-    return this.createTypeQueryTask("String", prompt);
+  async string(prompt, opt) {
+    return this.createTypeQueryTask("String", prompt, opt);
   }
-  async assert(assertion) {
+  async assert(assertion, memoryContext) {
     const description = `assert: ${assertion}`;
-    const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Assert", description), {
-      onTaskStart: this.onTaskStartCallback
-    });
+    const useMemory = true;
+    let taskExecutor;
+    if (useMemory) {
+      taskExecutor = this.getPersistentExecutor();
+    } else {
+      taskExecutor = new import_misoai_core.Executor(taskTitleStr("Assert", description), {
+        onTaskStart: this.onTaskStartCallback
+      });
+    }
     const assertionPlan = {
       type: "Assert",
       param: {
@@ -1587,7 +1879,7 @@ function buildPlans(type, locateParam, param) {
     param: locateParam,
     thought: ""
   } : null;
-  if (type === "Tap" || type === "Hover") {
+  if (type === "Tap" || type === "Hover" || type === "RightClick") {
     (0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
     (0, import_utils8.assert)(locatePlan, `missing locate info for action "${type}"`);
     const tapPlan = {
@@ -1667,7 +1959,7 @@ var import_js_yaml3 = __toESM(require("js-yaml"));
 var import_semver = __toESM(require("semver"));
 
 // package.json
-var version = "1.5.6";
+var version = "1.5.7";
 
 // src/common/task-cache.ts
 var debug3 = (0, import_logger3.getDebug)("cache");
@@ -1798,8 +2090,14 @@ cache file: ${cacheFile}`
       return;
     }
     try {
+      const dir = (0, import_node_path2.dirname)(this.cacheFilePath);
+      if (!(0, import_node_fs2.existsSync)(dir)) {
+        (0, import_node_fs2.mkdirSync)(dir, { recursive: true });
+        debug3("created cache directory: %s", dir);
+      }
       const yamlData = import_js_yaml3.default.dump(this.cache);
       (0, import_node_fs2.writeFileSync)(this.cacheFilePath, yamlData);
+      debug3("cache flushed to file: %s", this.cacheFilePath);
     } catch (err) {
       debug3(
         "write cache to file failed, path: %s, error: %s",
@@ -2057,6 +2355,23 @@ var PageAgent = class {
       metadata
     };
   }
+  async aiRightClick(locatePrompt, opt) {
+    const detailedLocateParam = this.buildDetailedLocateParam(
+      locatePrompt,
+      opt
+    );
+    const plans = buildPlans("RightClick", detailedLocateParam);
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("RightClick", locateParamStr(detailedLocateParam)),
+      plans,
+      { cacheable: opt?.cacheable }
+    );
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
   async aiInput(value, locatePrompt, opt) {
     (0, import_utils12.assert)(
       typeof value === "string",
@@ -2131,7 +2446,13 @@ var PageAgent = class {
         metadata: metadata2
       };
     }
-    const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable }) : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext, {
+    const memoryContext = this.getMemoryAsContext();
+    const enhancedActionContext = this.opts.aiActionContext ? `${this.opts.aiActionContext}
+
+Previous workflow steps:
+${memoryContext}` : memoryContext ? `Previous workflow steps:
+${memoryContext}` : void 0;
+    const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable }) : this.taskExecutor.action(taskPrompt, enhancedActionContext, {
       cacheable
     }));
     if (this.taskCache && output?.yamlFlow && cacheable !== false) {
@@ -2277,8 +2598,9 @@ var PageAgent = class {
       } catch (e) {
       }
     }
+    const memoryContext = this.getMemoryAsContext();
     const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
-    const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
+    const { output, executor } = await this.taskExecutor.assert(assertionWithContext, memoryContext);
     const metadata = this.afterTaskRunning(executor, true);
     if (output && opt?.keepRawResponse) {
       return {
@@ -2488,9 +2810,72 @@ ${errors}`);
     }
     throw new Error("evaluateJavaScript is not supported in current agent");
   }
+  async logScreenshot(title, options) {
+    const screenshotTitle = title || "untitled";
+    const content = options?.content || "";
+    const screenshot = await this.page.screenshotBase64?.();
+    if (screenshot) {
+      const executionDump = {
+        name: screenshotTitle,
+        description: content,
+        tasks: [{
+          type: "Screenshot",
+          subType: "log",
+          status: "finished",
+          executor: null,
+          param: {
+            title: screenshotTitle,
+            content
+          },
+          output: {
+            screenshot
+          },
+          thought: `Logged screenshot: ${screenshotTitle}`,
+          timing: {
+            start: Date.now(),
+            end: Date.now(),
+            cost: 0
+          }
+        }],
+        sdkVersion: "1.0.0",
+        logTime: Date.now(),
+        model_name: "screenshot"
+      };
+      this.appendExecutionDump(executionDump);
+    }
+  }
   async destroy() {
     await this.page.destroy();
   }
+  /**
+   * Hafızayı bağlam olarak formatlar
+   */
+  getMemoryAsContext() {
+    const memory = this.taskExecutor.getMemory();
+    if (memory.length === 0) {
+      return "";
+    }
+    const recentMemory = memory.slice(-5);
+    return recentMemory.map((item) => `- ${item.summary}`).join("\n");
+  }
+  /**
+   * Mevcut hafızayı döndürür
+   */
+  getMemory() {
+    return this.taskExecutor.getMemory();
+  }
+  /**
+   * Hafıza istatistiklerini döndürür
+   */
+  getMemoryStats() {
+    return this.taskExecutor.getMemoryStats();
+  }
+  /**
+   * Hafızayı temizler
+   */
+  clearMemory() {
+    this.taskExecutor.clearMemory();
+  }
 };
 
 // src/playground/agent.ts