misoai-web 1.0.5 → 1.0.6

package/bin/midscene-playground

@@ -1,3 +1,3 @@
-#!/usr/bin/env node
-
+#!/usr/bin/env node
+
 require('../dist/lib/midscene-playground.js');

package/dist/es/agent.js

@@ -1688,44 +1688,70 @@ var TaskCache = class {
     this.cache = cacheContent;
     this.cacheOriginalLength = this.cache.caches.length;
   }
-  matchCache(prompt, type) {
+  matchCache(prompt, type, contextData) {
+    const contextHash = contextData ? this.generateContextHash(contextData) : void 0;
     for (let i = 0; i < this.cacheOriginalLength; i++) {
       const item = this.cache.caches[i];
       const key = `${type}:${prompt}:${i}`;
-      if (item.type === type && item.prompt === prompt && !this.matchedCacheIndices.has(key)) {
-        this.matchedCacheIndices.add(key);
-        debug3(
-          "cache found and marked as used, type: %s, prompt: %s, index: %d",
-          type,
-          prompt,
-          i
-        );
-        return {
-          cacheContent: item,
-          updateFn: (cb) => {
-            debug3(
-              "will call updateFn to update cache, type: %s, prompt: %s, index: %d",
-              type,
-              prompt,
-              i
-            );
-            cb(item);
-            debug3(
-              "cache updated, will flush to file, type: %s, prompt: %s, index: %d",
-              type,
-              prompt,
-              i
-            );
-            this.flushCacheToFile();
+      if (item.type !== type || item.prompt !== prompt || this.matchedCacheIndices.has(key)) {
+        continue;
+      }
+      if (type === "plan" && item.type === "plan") {
+        const planItem = item;
+        if (contextHash && planItem.contextHash) {
+          if (contextHash !== planItem.contextHash) {
+            debug3("cache context mismatch, type: %s, prompt: %s, index: %d", type, prompt, i);
+            continue;
           }
-        };
+        } else if (contextHash || planItem.contextHash) {
+          debug3("cache context availability mismatch, type: %s, prompt: %s, index: %d", type, prompt, i);
+          continue;
+        }
       }
+      this.matchedCacheIndices.add(key);
+      debug3(
+        "cache found and marked as used, type: %s, prompt: %s, index: %d, contextMatch: %s",
+        type,
+        prompt,
+        i,
+        contextHash ? "yes" : "no-context"
+      );
+      return {
+        cacheContent: item,
+        updateFn: (cb) => {
+          debug3(
+            "will call updateFn to update cache, type: %s, prompt: %s, index: %d",
+            type,
+            prompt,
+            i
+          );
+          cb(item);
+          debug3(
+            "cache updated, will flush to file, type: %s, prompt: %s, index: %d",
+            type,
+            prompt,
+            i
+          );
+          this.flushCacheToFile();
+        }
+      };
     }
-    debug3("no unused cache found, type: %s, prompt: %s", type, prompt);
+    debug3("no unused cache found, type: %s, prompt: %s, contextHash: %s", type, prompt, contextHash);
     return void 0;
   }
-  matchPlanCache(prompt) {
-    return this.matchCache(prompt, "plan");
+  generateContextHash(contextData) {
+    const sortedKeys = Object.keys(contextData).sort();
+    const stableString = sortedKeys.map((key) => `${key}:${JSON.stringify(contextData[key])}`).join("|");
+    let hash = 0;
+    for (let i = 0; i < stableString.length; i++) {
+      const char = stableString.charCodeAt(i);
+      hash = (hash << 5) - hash + char;
+      hash = hash & hash;
+    }
+    return hash.toString(36);
+  }
+  matchPlanCache(prompt, contextData) {
+    return this.matchCache(prompt, "plan", contextData);
   }
   matchLocateCache(prompt) {
     return this.matchCache(prompt, "locate");
@@ -1801,11 +1827,16 @@ cache file: ${cacheFile}`
       );
     }
   }
-  updateOrAppendCacheRecord(newRecord, cachedRecord) {
+  updateOrAppendCacheRecord(newRecord, cachedRecord, contextData) {
     if (cachedRecord) {
       if (newRecord.type === "plan") {
         cachedRecord.updateFn((cache) => {
-          cache.yamlWorkflow = newRecord.yamlWorkflow;
+          const planCache = cache;
+          planCache.yamlWorkflow = newRecord.yamlWorkflow;
+          if (contextData) {
+            planCache.contextHash = this.generateContextHash(contextData);
+            planCache.contextData = { ...contextData };
+          }
         });
       } else {
         cachedRecord.updateFn((cache) => {
@@ -1813,6 +1844,11 @@ cache file: ${cacheFile}`
         });
       }
     } else {
+      if (newRecord.type === "plan" && contextData) {
+        const planRecord = newRecord;
+        planRecord.contextHash = this.generateContextHash(contextData);
+        planRecord.contextData = { ...contextData };
+      }
       this.appendCache(newRecord);
     }
   }
@@ -2173,34 +2209,35 @@ var PageAgent = class {
     };
   }
   async aiAction(taskPrompt, opt) {
+    const originalPrompt = taskPrompt;
+    let processedPrompt = taskPrompt;
     if (this.opts.enableCumulativeContext && this.contextStore) {
       try {
-        const originalPrompt = taskPrompt;
-        const processedPrompt = this.contextStore.replaceAllReferences(taskPrompt, "action");
-        if (originalPrompt !== processedPrompt) {
-          debug4("Context replacement in aiAction:", {
-            original: originalPrompt,
-            processed: processedPrompt,
-            storedData: this.contextStore.getAllData()
+        const storedData = this.contextStore.getAllData();
+        if (Object.keys(storedData).length > 0) {
+          debug4("Available data for aiAction:", {
+            prompt: taskPrompt,
+            availableData: storedData
           });
         }
-        this.contextStore.addStep({
-          type: "action",
-          summary: `Action: ${processedPrompt}`,
-          prompt: processedPrompt
-        });
-        debug4("Added action step to context store:", {
-          stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
-          totalSteps: this.contextStore.getRecentSteps(100).length
-        });
-        taskPrompt = processedPrompt;
       } catch (error) {
         debug4("Context store operation failed:", error);
       }
     }
     const cacheable = opt?.cacheable;
     const isVlmUiTars = vlLocateMode() === "vlm-ui-tars";
-    const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt);
+    let contextData;
+    if (this.opts.enableCumulativeContext && this.contextStore) {
+      try {
+        contextData = this.contextStore.getAllData();
+        if (contextData && Object.keys(contextData).length === 0) {
+          contextData = void 0;
+        }
+      } catch (error) {
+        debug4("Failed to get context data for cache:", error);
+      }
+    }
+    const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt, contextData);
     if (matchedCache && this.taskCache?.isCacheResultUsed) {
       const { executor: executor2 } = await this.taskExecutor.loadYamlFlowAsPlanning(
         taskPrompt,
@@ -2210,6 +2247,28 @@ var PageAgent = class {
       debug4("matched cache, will call .runYaml to run the action");
       const yaml5 = matchedCache.cacheContent?.yamlWorkflow;
       const result = await this.runYaml(yaml5);
+      if (this.opts.enableCumulativeContext && this.contextStore) {
+        try {
+          const executionResult = {
+            success: true,
+            actionType: "cached",
+            description: `Executed cached action: ${processedPrompt}`,
+            timing: result.metadata?.totalTime
+          };
+          this.contextStore.addStep({
+            type: "action",
+            summary: `Action: ${processedPrompt} (cached)`,
+            prompt: processedPrompt,
+            executionResult
+          });
+          debug4("Added cached action step to context store:", {
+            stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
+            totalSteps: this.contextStore.getRecentSteps(100).length
+          });
+        } catch (error) {
+          debug4("Failed to add cached action step:", error);
+        }
+      }
       return {
         result: result.result,
         metadata: metadata2
@@ -2234,10 +2293,39 @@ var PageAgent = class {
           prompt: taskPrompt,
           yamlWorkflow: yamlFlowStr
         },
-        matchedCache
+        matchedCache,
+        contextData
+        // Pass context data for cache creation
       );
     }
     const metadata = this.afterTaskRunning(executor);
+    if (this.opts.enableCumulativeContext && this.contextStore) {
+      try {
+        const executionResult = this.analyzeExecutionResults(executor, originalPrompt);
+        this.contextStore.addStep({
+          type: "action",
+          summary: `Action: ${processedPrompt}`,
+          prompt: processedPrompt,
+          executionResult
+        });
+        debug4("Added action step with execution result to context store:", {
+          stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
+          totalSteps: this.contextStore.getRecentSteps(100).length,
+          executionResult
+        });
+      } catch (error) {
+        debug4("Failed to analyze execution results, adding step without execution result:", error);
+        try {
+          this.contextStore.addStep({
+            type: "action",
+            summary: `Action: ${processedPrompt}`,
+            prompt: processedPrompt
+          });
+        } catch (stepError) {
+          debug4("Failed to add action step:", stepError);
+        }
+      }
+    }
     return {
       result: output,
       metadata
@@ -2422,23 +2510,40 @@ var PageAgent = class {
     };
   }
   async aiAssert(assertion, msg, opt) {
-    let processedAssertion = assertion;
+    let executionContext = "";
     if (this.opts.enableCumulativeContext && this.contextStore) {
       try {
-        const originalAssertion = assertion;
-        processedAssertion = this.contextStore.replaceAllReferences(assertion, "assertion");
-        if (originalAssertion !== processedAssertion) {
-          debug4("Context replacement in aiAssert:", {
-            original: originalAssertion,
-            processed: processedAssertion,
-            context: "assertion",
-            storedData: this.contextStore.getAllData()
+        const recentSteps = this.contextStore.getRecentSteps(3);
+        const stepsWithExecutionResults = recentSteps.filter((step) => step.executionResult);
+        const storedData = this.contextStore.getAllData();
+        if (stepsWithExecutionResults.length > 0) {
+          const recentActions = stepsWithExecutionResults.map((step) => {
+            const result = step.executionResult;
+            return `- ${result.description}${result.success ? "" : " (FAILED)"}`;
+          }).join("\n");
+          executionContext = `
+
+Recent actions performed:
+${recentActions}
+
+This context may help verify the assertion.`;
+        }
+        if (storedData && Object.keys(storedData).length > 0) {
+          executionContext += `
+
+Available data for reference:
+${JSON.stringify(storedData, null, 2)}
+
+Note: If the assertion references any data keys or natural language equivalents, consider the stored values when verifying.`;
+          debug4("Available data for aiAssert:", {
+            assertion,
+            availableData: storedData
           });
         }
         this.contextStore.addStep({
           type: "assertion",
-          summary: `Assertion: ${processedAssertion}`,
-          prompt: processedAssertion
+          summary: `Assertion: ${assertion}`,
+          prompt: assertion
         });
         debug4("Added assertion step to context store:", {
           totalSteps: this.contextStore.getRecentSteps(100).length
@@ -2454,7 +2559,13 @@ var PageAgent = class {
       } catch (e) {
       }
     }
-    const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${processedAssertion}` : processedAssertion;
+    let assertionWithContext = assertion;
+    if (currentUrl) {
+      assertionWithContext = `For the page at URL "${currentUrl}", ${assertion}`;
+    }
+    if (executionContext) {
+      assertionWithContext += executionContext;
+    }
     const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
     const metadata = this.afterTaskRunning(executor, true);
     if (output && opt?.keepRawResponse) {
@@ -2668,6 +2779,79 @@ ${errors}`);
   async destroy() {
     await this.page.destroy();
   }
+  /**
+   * Analyze execution results from executor to generate meaningful descriptions
+   */
+  analyzeExecutionResults(executor, originalPrompt) {
+    const tasks = executor.tasks;
+    const success = !executor.isInErrorState();
+    if (!success) {
+      const errorTask = executor.latestErrorTask();
+      return {
+        success: false,
+        actionType: "error",
+        description: `Failed to execute: ${originalPrompt}`,
+        error: errorTask?.error
+      };
+    }
+    const actionTasks = tasks.filter((t) => t.type === "Action" && t.status === "finished");
+    const locateTasks = tasks.filter((t) => t.type === "Insight" && t.subType === "Locate");
+    const lastAction = actionTasks[actionTasks.length - 1];
+    const lastLocate = locateTasks[locateTasks.length - 1];
+    if (!lastAction) {
+      return {
+        success: true,
+        actionType: "unknown",
+        description: `Completed: ${originalPrompt}`
+      };
+    }
+    const actionType = lastAction.subType || "unknown";
+    const elementInfo = this.extractElementInfo(lastLocate, lastAction);
+    const description = this.generateActionDescription(actionType, lastAction.param, elementInfo);
+    return {
+      success: true,
+      actionType,
+      description,
+      elementInfo,
+      timing: lastAction.timing?.cost
+    };
+  }
+  /**
+   * Extract element information from locate task
+   */
+  extractElementInfo(locateTask, _actionTask) {
+    if (!locateTask?.output?.element)
+      return void 0;
+    const element = locateTask.output.element;
+    return {
+      type: element.attributes?.nodeType || "unknown",
+      text: element.content || element.attributes?.placeholder || element.attributes?.title || "",
+      location: `(${element.center[0]}, ${element.center[1]})`
+    };
+  }
+  /**
+   * Generate natural language description for actions
+   */
+  generateActionDescription(actionType, param, elementInfo) {
+    const elementDesc = elementInfo ? `'${elementInfo.text || elementInfo.type}' element` : "element";
+    switch (actionType) {
+      case "Tap":
+        return `Clicked on ${elementDesc}`;
+      case "Input":
+        const inputValue = param?.value || "";
+        return `Entered "${inputValue}" into ${elementDesc}`;
+      case "KeyboardPress":
+        return `Pressed ${param?.value || "key"}`;
+      case "Scroll":
+        return `Scrolled ${param?.direction || "on page"}`;
+      case "Hover":
+        return `Hovered over ${elementDesc}`;
+      case "Drag":
+        return `Dragged ${elementDesc}`;
+      default:
+        return `Performed ${actionType} action on ${elementDesc}`;
+    }
+  }
 };
 export {
   PageAgent