misoai-web 1.5.6 → 1.5.8

package/dist/es/midscene-playground.js

@@ -22,7 +22,8 @@ var WebElementInfo = class {
     id,
     attributes,
     indexId,
-    xpaths
+    xpaths,
+    isVisible
   }) {
     this.content = content;
     this.rect = rect;
@@ -35,6 +36,7 @@ var WebElementInfo = class {
     this.attributes = attributes;
     this.indexId = indexId;
     this.xpaths = xpaths;
+    this.isVisible = isVisible;
   }
 };
 
@@ -57,14 +59,15 @@ async function parseContextFromWebPage(page, _opt) {
     })
   ]);
   const webTree = traverseTree(tree, (elementInfo) => {
-    const { rect, id, content, attributes, locator, indexId } = elementInfo;
+    const { rect, id, content, attributes, locator, indexId, isVisible } = elementInfo;
     return new WebElementInfo({
       rect,
       locator,
       id,
       content,
       attributes,
-      indexId
+      indexId,
+      isVisible
     });
   });
   assert(screenshotBase64, "screenshotBase64 is required");
@@ -95,7 +98,7 @@ function printReportMsg(filepath) {
 }
 var ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED = "NOT_IMPLEMENTED_AS_DESIGNED";
 function replaceIllegalPathCharsAndSpace(str) {
-  return str.replace(/[/\\:*?"<>| ]/g, "-");
+  return str.replace(/[:*?"<>| ]/g, "-");
 }
 function matchElementFromPlan(planLocateParam, tree) {
   if (!planLocateParam) {
@@ -225,6 +228,10 @@ var ScriptPlayer = class {
       } else if ("aiQuery" in flowItem) {
         const queryTask = flowItem;
         const prompt = queryTask.aiQuery;
+        const options = {
+          domIncluded: queryTask.domIncluded,
+          screenshotIncluded: queryTask.screenshotIncluded
+        };
         assert2(prompt, "missing prompt for aiQuery");
         assert2(
           typeof prompt === "string",
@@ -235,6 +242,10 @@ var ScriptPlayer = class {
       } else if ("aiNumber" in flowItem) {
         const numberTask = flowItem;
         const prompt = numberTask.aiNumber;
+        const options = {
+          domIncluded: numberTask.domIncluded,
+          screenshotIncluded: numberTask.screenshotIncluded
+        };
         assert2(prompt, "missing prompt for number");
         assert2(
           typeof prompt === "string",
@@ -245,6 +256,10 @@ var ScriptPlayer = class {
       } else if ("aiString" in flowItem) {
         const stringTask = flowItem;
         const prompt = stringTask.aiString;
+        const options = {
+          domIncluded: stringTask.domIncluded,
+          screenshotIncluded: stringTask.screenshotIncluded
+        };
         assert2(prompt, "missing prompt for string");
         assert2(
           typeof prompt === "string",
@@ -255,6 +270,10 @@ var ScriptPlayer = class {
       } else if ("aiBoolean" in flowItem) {
         const booleanTask = flowItem;
         const prompt = booleanTask.aiBoolean;
+        const options = {
+          domIncluded: booleanTask.domIncluded,
+          screenshotIncluded: booleanTask.screenshotIncluded
+        };
         assert2(prompt, "missing prompt for boolean");
         assert2(
           typeof prompt === "string",
@@ -297,6 +316,9 @@ var ScriptPlayer = class {
       } else if ("aiTap" in flowItem) {
         const tapTask = flowItem;
         await agent.aiTap(tapTask.aiTap, tapTask);
+      } else if ("aiRightClick" in flowItem) {
+        const rightClickTask = flowItem;
+        await agent.aiRightClick(rightClickTask.aiRightClick, rightClickTask);
       } else if ("aiHover" in flowItem) {
         const hoverTask = flowItem;
         await agent.aiHover(hoverTask.aiHover, hoverTask);
@@ -319,6 +341,11 @@ var ScriptPlayer = class {
           evaluateJavaScriptTask.javascript
         );
         this.setResult(evaluateJavaScriptTask.name, result);
+      } else if ("logScreenshot" in flowItem) {
+        const logScreenshotTask = flowItem;
+        await agent.logScreenshot(logScreenshotTask.logScreenshot, {
+          content: logScreenshotTask.content || ""
+        });
       } else {
         throw new Error(`unknown flowItem: ${JSON.stringify(flowItem)}`);
       }
@@ -569,6 +596,94 @@ var replanningCountLimit = 10;
 var isAndroidPage = (page) => {
   return page.pageType === "android";
 };
+var WorkflowMemory = class {
+  constructor(config) {
+    this.workflows = /* @__PURE__ */ new Map();
+    this.config = config;
+  }
+  /**
+   * İş akışı hafızasını getirir
+   */
+  getWorkflowMemory(workflowId = "default") {
+    const workflow = this.workflows.get(workflowId);
+    return workflow?.memory || [];
+  }
+  /**
+   * İş akışı verilerini getirir
+   */
+  getWorkflowData(workflowId = "default") {
+    return this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
+  }
+  /**
+   * İş akışı hafızasını kaydeder
+   */
+  saveWorkflowMemory(memory, workflowId = "default") {
+    const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
+    workflow.memory = [...memory];
+    workflow.metadata.totalSteps = workflow.steps.length;
+    workflow.metadata.completedSteps = workflow.steps.filter((s) => s.status === "completed").length;
+    workflow.metadata.failedSteps = workflow.steps.filter((s) => s.status === "failed").length;
+    this.workflows.set(workflowId, workflow);
+    this.enforceRetentionPolicy();
+  }
+  /**
+   * İş akışı bağlamını günceller
+   */
+  updateWorkflowContext(context, workflowId = "default") {
+    const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
+    workflow.context = { ...workflow.context, ...context };
+    if (context.currentStep) {
+      const existingStep = workflow.steps.find((s) => s.stepName === context.currentStep);
+      if (!existingStep) {
+        workflow.steps.push({
+          stepId: `step_${workflow.steps.length + 1}`,
+          stepName: context.currentStep,
+          timestamp: context.timestamp,
+          status: "running",
+          memoryItems: []
+        });
+      }
+    }
+    this.workflows.set(workflowId, workflow);
+  }
+  /**
+   * İş akışını temizler
+   */
+  clearWorkflow(workflowId = "default") {
+    this.workflows.delete(workflowId);
+  }
+  /**
+   * Tüm iş akışlarını temizler
+   */
+  clearAll() {
+    this.workflows.clear();
+  }
+  createEmptyWorkflowData(workflowId) {
+    return {
+      workflowId,
+      steps: [],
+      memory: [],
+      context: {
+        pageInfo: { url: "", title: "" },
+        timestamp: Date.now()
+      },
+      metadata: {
+        totalSteps: 0,
+        completedSteps: 0,
+        failedSteps: 0,
+        startTime: Date.now()
+      }
+    };
+  }
+  enforceRetentionPolicy() {
+    const maxWorkflows = 10;
+    if (this.workflows.size > maxWorkflows) {
+      const sortedWorkflows = Array.from(this.workflows.entries()).sort(([, a], [, b]) => (b.metadata.endTime || b.metadata.startTime) - (a.metadata.endTime || a.metadata.startTime));
+      const toDelete = sortedWorkflows.slice(maxWorkflows);
+      toDelete.forEach(([workflowId]) => this.workflows.delete(workflowId));
+    }
+  }
+};
 var PageTaskExecutor = class {
   constructor(page, insight, opts) {
     this.conversationHistory = [];
@@ -576,6 +691,25 @@ var PageTaskExecutor = class {
     this.insight = insight;
     this.taskCache = opts.taskCache;
     this.onTaskStartCallback = opts?.onTaskStart;
+    this.memoryConfig = {
+      maxItems: 100,
+      maxAge: 2 * 60 * 60 * 1e3,
+      // 2 saat
+      enablePersistence: true,
+      enableAnalytics: true,
+      filterStrategy: "hybrid",
+      ...opts?.memoryConfig
+    };
+    this.sessionContext = {
+      sessionId: opts?.sessionId || this.generateSessionId(),
+      workflowId: opts?.workflowId,
+      startTime: Date.now(),
+      pageInfo: {
+        url: "",
+        title: ""
+      }
+    };
+    this.workflowMemory = new WorkflowMemory(this.memoryConfig);
   }
   async recordScreenshot(timing) {
     const base64 = await this.page.screenshotBase64();
@@ -788,8 +922,11 @@ var PageTaskExecutor = class {
               insightDump = dump;
             };
             this.insight.onceDumpUpdatedFn = dumpCollector;
+            const memoryContext = this.getMemoryAsContext();
             const assertion = await this.insight.assert(
-              assertPlan.param.assertion
+              assertPlan.param.assertion,
+              memoryContext
+              // Hafıza bağlamını geç
             );
             if (!assertion.pass) {
               if (plan2.type === "Assert") {
@@ -826,10 +963,10 @@ var PageTaskExecutor = class {
               if (!taskParam || !taskParam.value) {
                 return;
               }
-              await this.page.keyboard.type(taskParam.value);
-            } else {
-              await this.page.keyboard.type(taskParam.value);
             }
+            await this.page.keyboard.type(taskParam.value, {
+              autoDismissKeyboard: taskParam.autoDismissKeyboard
+            });
           }
         };
         tasks.push(taskActionInput);
@@ -858,6 +995,22 @@ var PageTaskExecutor = class {
           }
         };
         tasks.push(taskActionTap);
+      } else if (plan2.type === "RightClick") {
+        const taskActionRightClick = {
+          type: "Action",
+          subType: "RightClick",
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (param, { element }) => {
+            assert4(element, "Element not found, cannot right click");
+            await this.page.mouse.click(
+              element.center[0],
+              element.center[1],
+              { button: "right" }
+            );
+          }
+        };
+        tasks.push(taskActionRightClick);
       } else if (plan2.type === "Drag") {
         const taskActionDrag = {
           type: "Action",
@@ -1259,25 +1412,146 @@ var PageTaskExecutor = class {
     };
     return task;
   }
+  /**
+   * Persistent executor'ı getirir veya oluşturur
+   */
+  getPersistentExecutor() {
+    if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
+      const previousMemory = this.workflowMemory.getWorkflowMemory(this.sessionContext.workflowId);
+      this.persistentExecutor = new Executor("Persistent Task Executor", {
+        onTaskStart: this.onTaskStartCallback,
+        initialMemory: previousMemory
+      });
+    }
+    return this.persistentExecutor;
+  }
+  /**
+   * Sayfa bağlamını günceller
+   */
+  async updatePageContext() {
+    try {
+      if (this.page.url) {
+        this.sessionContext.pageInfo.url = await this.page.url();
+      }
+      if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
+        this.sessionContext.pageInfo.title = await this.page.title();
+      }
+    } catch (e) {
+    }
+  }
+  /**
+   * Hafızayı temizler
+   */
+  clearMemory() {
+    if (this.persistentExecutor) {
+      this.persistentExecutor.clearMemory();
+    }
+    this.workflowMemory.clearWorkflow(this.sessionContext.workflowId || "default");
+  }
+  /**
+   * Mevcut hafızayı döndürür
+   */
+  getMemory() {
+    return this.persistentExecutor?.getMemory() || [];
+  }
+  /**
+   * İş akışı hafızasını döndürür
+   */
+  getWorkflowMemory() {
+    return this.workflowMemory.getWorkflowData(this.sessionContext.workflowId || "default");
+  }
+  /**
+   * Hafıza istatistiklerini döndürür
+   */
+  getMemoryStats() {
+    return this.persistentExecutor?.getMemoryStats() || {
+      totalItems: 0,
+      analytics: { totalTasks: 0, memoryHits: 0, memoryMisses: 0, averageMemorySize: 0, memoryEffectiveness: 0 },
+      config: this.memoryConfig
+    };
+  }
+  /**
+   * Hafıza konfigürasyonunu günceller
+   */
+  updateMemoryConfig(config) {
+    this.memoryConfig = { ...this.memoryConfig, ...config };
+    if (this.persistentExecutor) {
+      this.persistentExecutor.updateMemoryConfig(this.memoryConfig);
+    }
+  }
+  /**
+   * Oturum ID'sini oluşturur
+   */
+  generateSessionId() {
+    return `session_${Date.now()}_${Math.random().toString(36).substr(2, 8)}`;
+  }
+  /**
+   * Hafızayı bağlam olarak formatlar
+   */
+  getMemoryAsContext() {
+    const memory = this.getMemory();
+    if (memory.length === 0) {
+      return "";
+    }
+    const recentMemory = memory.slice(-5);
+    return recentMemory.map((item) => `- ${item.summary}`).join("\n");
+  }
   async runPlans(title, plans, opts) {
-    const taskExecutor = new Executor(title, {
-      onTaskStart: this.onTaskStartCallback
-    });
+    await this.updatePageContext();
+    const useMemory = opts?.useMemory !== false;
+    let taskExecutor;
+    if (useMemory) {
+      taskExecutor = this.getPersistentExecutor();
+      this.workflowMemory.updateWorkflowContext({
+        currentStep: title,
+        pageInfo: this.sessionContext.pageInfo,
+        timestamp: Date.now()
+      }, this.sessionContext.workflowId || "default");
+    } else {
+      taskExecutor = new Executor(title, {
+        onTaskStart: this.onTaskStartCallback
+      });
+    }
     const { tasks } = await this.convertPlanToExecutable(plans, opts);
+    tasks.forEach((task) => {
+      task.context = {
+        ...task.context,
+        ...this.sessionContext.pageInfo,
+        workflowId: this.sessionContext.workflowId,
+        sessionId: this.sessionContext.sessionId
+      };
+    });
     await taskExecutor.append(tasks);
     const result = await taskExecutor.flush();
+    if (useMemory) {
+      this.workflowMemory.saveWorkflowMemory(
+        taskExecutor.getMemory(),
+        this.sessionContext.workflowId || "default"
+      );
+    }
     return {
       output: result,
       executor: taskExecutor
     };
   }
   async action(userPrompt, actionContext, opts) {
-    const taskExecutor = new Executor(taskTitleStr("Action", userPrompt), {
-      onTaskStart: this.onTaskStartCallback
-    });
-    let planningTask = this.planningTaskFromPrompt(userPrompt, void 0, actionContext);
+    const useMemory = true;
+    let taskExecutor;
+    if (useMemory) {
+      taskExecutor = this.getPersistentExecutor();
+    } else {
+      taskExecutor = new Executor(taskTitleStr("Action", userPrompt), {
+        onTaskStart: this.onTaskStartCallback
+      });
+    }
+    const memoryContext = this.getMemoryAsContext();
+    const initialLog = memoryContext ? memoryContext : void 0;
+    let planningTask = this.planningTaskFromPrompt(userPrompt, initialLog, actionContext);
     let replanCount = 0;
     const logList = [];
+    if (memoryContext) {
+      logList.push(memoryContext);
+    }
     const yamlFlow = [];
     while (planningTask) {
       if (replanCount > replanningCountLimit) {
@@ -1386,16 +1660,22 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async createTypeQueryTask(type, demand) {
-    const taskExecutor = new Executor(
-      taskTitleStr(
-        type,
-        typeof demand === "string" ? demand : JSON.stringify(demand)
-      ),
-      {
-        onTaskStart: this.onTaskStartCallback
-      }
-    );
+  async createTypeQueryTask(type, demand, opt) {
+    const useMemory = true;
+    let taskExecutor;
+    if (useMemory) {
+      taskExecutor = this.getPersistentExecutor();
+    } else {
+      taskExecutor = new Executor(
+        taskTitleStr(
+          type,
+          typeof demand === "string" ? demand : JSON.stringify(demand)
+        ),
+        {
+          onTaskStart: this.onTaskStartCallback
+        }
+      );
+    }
     const queryTask = {
       type: "Insight",
       subType: type,
@@ -1417,7 +1697,13 @@ var PageTaskExecutor = class {
             result: `${type}, ${demand}`
           };
         }
-        const { data, usage } = await this.insight.extract(demandInput);
+        const memoryContext = this.getMemoryAsContext();
+        const { data, usage } = await this.insight.extract(
+          demandInput,
+          opt,
+          memoryContext
+          // Hafıza bağlamını geç
+        );
         let outputResult = data;
         if (ifTypeRestricted) {
           assert4(data?.result !== void 0, "No result in query data");
@@ -1437,23 +1723,29 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async query(demand) {
-    return this.createTypeQueryTask("Query", demand);
+  async query(demand, opt) {
+    return this.createTypeQueryTask("Query", demand, opt);
   }
-  async boolean(prompt) {
-    return this.createTypeQueryTask("Boolean", prompt);
+  async boolean(prompt, opt) {
+    return this.createTypeQueryTask("Boolean", prompt, opt);
   }
-  async number(prompt) {
-    return this.createTypeQueryTask("Number", prompt);
+  async number(prompt, opt) {
+    return this.createTypeQueryTask("Number", prompt, opt);
   }
-  async string(prompt) {
-    return this.createTypeQueryTask("String", prompt);
+  async string(prompt, opt) {
+    return this.createTypeQueryTask("String", prompt, opt);
   }
-  async assert(assertion) {
+  async assert(assertion, memoryContext) {
     const description = `assert: ${assertion}`;
-    const taskExecutor = new Executor(taskTitleStr("Assert", description), {
-      onTaskStart: this.onTaskStartCallback
-    });
+    const useMemory = true;
+    let taskExecutor;
+    if (useMemory) {
+      taskExecutor = this.getPersistentExecutor();
+    } else {
+      taskExecutor = new Executor(taskTitleStr("Assert", description), {
+        onTaskStart: this.onTaskStartCallback
+      });
+    }
     const assertionPlan = {
       type: "Assert",
       param: {
@@ -1583,7 +1875,7 @@ function buildPlans(type, locateParam, param) {
     param: locateParam,
     thought: ""
   } : null;
-  if (type === "Tap" || type === "Hover") {
+  if (type === "Tap" || type === "Hover" || type === "RightClick") {
     assert5(locateParam, `missing locate info for action "${type}"`);
     assert5(locatePlan, `missing locate info for action "${type}"`);
     const tapPlan = {
@@ -1654,8 +1946,8 @@ function buildPlans(type, locateParam, param) {
 
 // src/common/task-cache.ts
 import assert6 from "assert";
-import { existsSync as existsSync2, readFileSync, writeFileSync as writeFileSync2 } from "fs";
-import { join as join2 } from "path";
+import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync, writeFileSync as writeFileSync2 } from "fs";
+import { dirname as dirname2, join as join2 } from "path";
 import { getMidsceneRunSubDir as getMidsceneRunSubDir2 } from "misoai-shared/common";
 import { getDebug as getDebug3 } from "misoai-shared/logger";
 import { ifInBrowser as ifInBrowser2 } from "misoai-shared/utils";
@@ -1663,7 +1955,7 @@ import yaml3 from "js-yaml";
 import semver from "semver";
 
 // package.json
-var version = "1.5.6";
+var version = "1.5.7";
 
 // src/common/task-cache.ts
 var debug3 = getDebug3("cache");
@@ -1794,8 +2086,14 @@ cache file: ${cacheFile}`
       return;
     }
     try {
+      const dir = dirname2(this.cacheFilePath);
+      if (!existsSync2(dir)) {
+        mkdirSync2(dir, { recursive: true });
+        debug3("created cache directory: %s", dir);
+      }
       const yamlData = yaml3.dump(this.cache);
       writeFileSync2(this.cacheFilePath, yamlData);
+      debug3("cache flushed to file: %s", this.cacheFilePath);
     } catch (err) {
       debug3(
         "write cache to file failed, path: %s, error: %s",
@@ -2053,6 +2351,23 @@ var PageAgent = class {
       metadata
     };
   }
+  async aiRightClick(locatePrompt, opt) {
+    const detailedLocateParam = this.buildDetailedLocateParam(
+      locatePrompt,
+      opt
+    );
+    const plans = buildPlans("RightClick", detailedLocateParam);
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("RightClick", locateParamStr(detailedLocateParam)),
+      plans,
+      { cacheable: opt?.cacheable }
+    );
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
   async aiInput(value, locatePrompt, opt) {
     assert7(
       typeof value === "string",
@@ -2127,7 +2442,13 @@ var PageAgent = class {
         metadata: metadata2
       };
     }
-    const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable }) : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext, {
+    const memoryContext = this.getMemoryAsContext();
+    const enhancedActionContext = this.opts.aiActionContext ? `${this.opts.aiActionContext}
+
+Previous workflow steps:
+${memoryContext}` : memoryContext ? `Previous workflow steps:
+${memoryContext}` : void 0;
+    const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable }) : this.taskExecutor.action(taskPrompt, enhancedActionContext, {
       cacheable
     }));
     if (this.taskCache && output?.yamlFlow && cacheable !== false) {
@@ -2273,8 +2594,9 @@ var PageAgent = class {
       } catch (e) {
       }
     }
+    const memoryContext = this.getMemoryAsContext();
     const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
-    const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
+    const { output, executor } = await this.taskExecutor.assert(assertionWithContext, memoryContext);
     const metadata = this.afterTaskRunning(executor, true);
     if (output && opt?.keepRawResponse) {
       return {
@@ -2484,9 +2806,72 @@ ${errors}`);
     }
     throw new Error("evaluateJavaScript is not supported in current agent");
   }
+  async logScreenshot(title, options) {
+    const screenshotTitle = title || "untitled";
+    const content = options?.content || "";
+    const screenshot = await this.page.screenshotBase64?.();
+    if (screenshot) {
+      const executionDump = {
+        name: screenshotTitle,
+        description: content,
+        tasks: [{
+          type: "Screenshot",
+          subType: "log",
+          status: "finished",
+          executor: null,
+          param: {
+            title: screenshotTitle,
+            content
+          },
+          output: {
+            screenshot
+          },
+          thought: `Logged screenshot: ${screenshotTitle}`,
+          timing: {
+            start: Date.now(),
+            end: Date.now(),
+            cost: 0
+          }
+        }],
+        sdkVersion: "1.0.0",
+        logTime: Date.now(),
+        model_name: "screenshot"
+      };
+      this.appendExecutionDump(executionDump);
+    }
+  }
   async destroy() {
     await this.page.destroy();
   }
+  /**
+   * Hafızayı bağlam olarak formatlar
+   */
+  getMemoryAsContext() {
+    const memory = this.taskExecutor.getMemory();
+    if (memory.length === 0) {
+      return "";
+    }
+    const recentMemory = memory.slice(-5);
+    return recentMemory.map((item) => `- ${item.summary}`).join("\n");
+  }
+  /**
+   * Mevcut hafızayı döndürür
+   */
+  getMemory() {
+    return this.taskExecutor.getMemory();
+  }
+  /**
+   * Hafıza istatistiklerini döndürür
+   */
+  getMemoryStats() {
+    return this.taskExecutor.getMemoryStats();
+  }
+  /**
+   * Hafızayı temizler
+   */
+  clearMemory() {
+    this.taskExecutor.clearMemory();
+  }
 };
 
 // src/playground/agent.ts