misoai-web 1.0.4 → 1.0.6

package/dist/es/midscene-playground.js

@@ -1661,7 +1661,7 @@ import yaml3 from "js-yaml";
 import semver from "semver";
 
 // package.json
-var version = "1.0.4";
+var version = "1.0.5";
 
 // src/common/task-cache.ts
 var debug3 = getDebug3("cache");
@@ -1689,44 +1689,70 @@ var TaskCache = class {
     this.cache = cacheContent;
     this.cacheOriginalLength = this.cache.caches.length;
   }
-  matchCache(prompt, type) {
+  matchCache(prompt, type, contextData) {
+    const contextHash = contextData ? this.generateContextHash(contextData) : void 0;
     for (let i = 0; i < this.cacheOriginalLength; i++) {
       const item = this.cache.caches[i];
       const key = `${type}:${prompt}:${i}`;
-      if (item.type === type && item.prompt === prompt && !this.matchedCacheIndices.has(key)) {
-        this.matchedCacheIndices.add(key);
-        debug3(
-          "cache found and marked as used, type: %s, prompt: %s, index: %d",
-          type,
-          prompt,
-          i
-        );
-        return {
-          cacheContent: item,
-          updateFn: (cb) => {
-            debug3(
-              "will call updateFn to update cache, type: %s, prompt: %s, index: %d",
-              type,
-              prompt,
-              i
-            );
-            cb(item);
-            debug3(
-              "cache updated, will flush to file, type: %s, prompt: %s, index: %d",
-              type,
-              prompt,
-              i
-            );
-            this.flushCacheToFile();
+      if (item.type !== type || item.prompt !== prompt || this.matchedCacheIndices.has(key)) {
+        continue;
+      }
+      if (type === "plan" && item.type === "plan") {
+        const planItem = item;
+        if (contextHash && planItem.contextHash) {
+          if (contextHash !== planItem.contextHash) {
+            debug3("cache context mismatch, type: %s, prompt: %s, index: %d", type, prompt, i);
+            continue;
           }
-        };
+        } else if (contextHash || planItem.contextHash) {
+          debug3("cache context availability mismatch, type: %s, prompt: %s, index: %d", type, prompt, i);
+          continue;
+        }
       }
+      this.matchedCacheIndices.add(key);
+      debug3(
+        "cache found and marked as used, type: %s, prompt: %s, index: %d, contextMatch: %s",
+        type,
+        prompt,
+        i,
+        contextHash ? "yes" : "no-context"
+      );
+      return {
+        cacheContent: item,
+        updateFn: (cb) => {
+          debug3(
+            "will call updateFn to update cache, type: %s, prompt: %s, index: %d",
+            type,
+            prompt,
+            i
+          );
+          cb(item);
+          debug3(
+            "cache updated, will flush to file, type: %s, prompt: %s, index: %d",
+            type,
+            prompt,
+            i
+          );
+          this.flushCacheToFile();
+        }
+      };
     }
-    debug3("no unused cache found, type: %s, prompt: %s", type, prompt);
+    debug3("no unused cache found, type: %s, prompt: %s, contextHash: %s", type, prompt, contextHash);
     return void 0;
   }
-  matchPlanCache(prompt) {
-    return this.matchCache(prompt, "plan");
+  generateContextHash(contextData) {
+    const sortedKeys = Object.keys(contextData).sort();
+    const stableString = sortedKeys.map((key) => `${key}:${JSON.stringify(contextData[key])}`).join("|");
+    let hash = 0;
+    for (let i = 0; i < stableString.length; i++) {
+      const char = stableString.charCodeAt(i);
+      hash = (hash << 5) - hash + char;
+      hash = hash & hash;
+    }
+    return hash.toString(36);
+  }
+  matchPlanCache(prompt, contextData) {
+    return this.matchCache(prompt, "plan", contextData);
   }
   matchLocateCache(prompt) {
     return this.matchCache(prompt, "locate");
@@ -1802,11 +1828,16 @@ cache file: ${cacheFile}`
       );
     }
   }
-  updateOrAppendCacheRecord(newRecord, cachedRecord) {
+  updateOrAppendCacheRecord(newRecord, cachedRecord, contextData) {
     if (cachedRecord) {
       if (newRecord.type === "plan") {
         cachedRecord.updateFn((cache) => {
-          cache.yamlWorkflow = newRecord.yamlWorkflow;
+          const planCache = cache;
+          planCache.yamlWorkflow = newRecord.yamlWorkflow;
+          if (contextData) {
+            planCache.contextHash = this.generateContextHash(contextData);
+            planCache.contextData = { ...contextData };
+          }
         });
       } else {
         cachedRecord.updateFn((cache) => {
@@ -1814,6 +1845,11 @@ cache file: ${cacheFile}`
         });
       }
     } else {
+      if (newRecord.type === "plan" && contextData) {
+        const planRecord = newRecord;
+        planRecord.contextHash = this.generateContextHash(contextData);
+        planRecord.contextData = { ...contextData };
+      }
       this.appendCache(newRecord);
     }
   }
@@ -1843,10 +1879,13 @@ var PageAgent = class {
         generateReport: true,
         autoPrintReportMsg: true,
         groupName: "Midscene Report",
-        groupDescription: ""
+        groupDescription: "",
+        enableCumulativeContext: true,
+        autoClearContext: false
       },
       opts || {}
     );
+    this.initializeContextStore();
     if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
       this.page.waitForNavigationTimeout = this.opts.waitForNavigationTimeout || DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
       this.page.waitForNetworkIdleTimeout = this.opts.waitForNetworkIdleTimeout || DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT;
@@ -1873,6 +1912,69 @@ var PageAgent = class {
       opts?.testId || this.page.pageType || "web"
     );
   }
+  /**
+   * Initialize context store for cumulative context functionality
+   */
+  async initializeContextStore() {
+    if (!this.opts.enableCumulativeContext) {
+      debug4("Cumulative context disabled via options");
+      return;
+    }
+    try {
+      const aiModel = await import("misoai-core/ai-model");
+      this.contextStore = aiModel.getContextStore();
+      debug4("Context store initialized successfully", {
+        autoClearContext: this.opts.autoClearContext,
+        testId: this.opts.testId
+      });
+      if (this.opts.autoClearContext) {
+        this.contextStore.clear();
+        debug4("Context store cleared due to autoClearContext option");
+      } else {
+        const existingData = this.contextStore.getAllData();
+        const existingSteps = this.contextStore.getRecentSteps(100).length;
+        debug4("Context store preserving existing data", {
+          existingDataKeys: Object.keys(existingData),
+          existingStepsCount: existingSteps
+        });
+      }
+    } catch (error) {
+      debug4("Failed to initialize context store:", error);
+      console.warn("⚠️ Could not initialize context store:", error);
+    }
+  }
+  /**
+   * Get the context store instance
+   */
+  getContextStore() {
+    return this.contextStore;
+  }
+  /**
+   * Clear the context store
+   */
+  clearContext() {
+    if (this.contextStore) {
+      this.contextStore.clear();
+    }
+  }
+  /**
+   * Get all stored data from context store
+   */
+  getStoredData() {
+    if (this.contextStore) {
+      return this.contextStore.getAllData();
+    }
+    return {};
+  }
+  /**
+   * Get step summary from context store
+   */
+  getStepSummary() {
+    if (this.contextStore) {
+      return this.contextStore.getStepSummary();
+    }
+    return "";
+  }
   async getUIContext(action) {
     if (action && (action === "extract" || action === "assert" || action === "captcha")) {
       return await parseContextFromWebPage(this.page, {
@@ -2108,22 +2210,35 @@ var PageAgent = class {
     };
   }
   async aiAction(taskPrompt, opt) {
-    try {
-      const aiModel = await import("misoai-core/ai-model");
-      const contextStore = aiModel.getContextStore();
-      const processedPrompt = contextStore.replaceAllReferences(taskPrompt, "action");
-      contextStore.addStep({
-        type: "action",
-        summary: `Action: ${processedPrompt}`,
-        prompt: processedPrompt
-      });
-      taskPrompt = processedPrompt;
-    } catch (error) {
-      debug4("Context store not available:", error);
+    const originalPrompt = taskPrompt;
+    let processedPrompt = taskPrompt;
+    if (this.opts.enableCumulativeContext && this.contextStore) {
+      try {
+        const storedData = this.contextStore.getAllData();
+        if (Object.keys(storedData).length > 0) {
+          debug4("Available data for aiAction:", {
+            prompt: taskPrompt,
+            availableData: storedData
+          });
+        }
+      } catch (error) {
+        debug4("Context store operation failed:", error);
+      }
     }
     const cacheable = opt?.cacheable;
     const isVlmUiTars = vlLocateMode() === "vlm-ui-tars";
-    const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt);
+    let contextData;
+    if (this.opts.enableCumulativeContext && this.contextStore) {
+      try {
+        contextData = this.contextStore.getAllData();
+        if (contextData && Object.keys(contextData).length === 0) {
+          contextData = void 0;
+        }
+      } catch (error) {
+        debug4("Failed to get context data for cache:", error);
+      }
+    }
+    const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt, contextData);
     if (matchedCache && this.taskCache?.isCacheResultUsed) {
       const { executor: executor2 } = await this.taskExecutor.loadYamlFlowAsPlanning(
         taskPrompt,
@@ -2133,6 +2248,28 @@ var PageAgent = class {
       debug4("matched cache, will call .runYaml to run the action");
       const yaml5 = matchedCache.cacheContent?.yamlWorkflow;
       const result = await this.runYaml(yaml5);
+      if (this.opts.enableCumulativeContext && this.contextStore) {
+        try {
+          const executionResult = {
+            success: true,
+            actionType: "cached",
+            description: `Executed cached action: ${processedPrompt}`,
+            timing: result.metadata?.totalTime
+          };
+          this.contextStore.addStep({
+            type: "action",
+            summary: `Action: ${processedPrompt} (cached)`,
+            prompt: processedPrompt,
+            executionResult
+          });
+          debug4("Added cached action step to context store:", {
+            stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
+            totalSteps: this.contextStore.getRecentSteps(100).length
+          });
+        } catch (error) {
+          debug4("Failed to add cached action step:", error);
+        }
+      }
       return {
         result: result.result,
         metadata: metadata2
@@ -2157,10 +2294,39 @@ var PageAgent = class {
           prompt: taskPrompt,
           yamlWorkflow: yamlFlowStr
         },
-        matchedCache
+        matchedCache,
+        contextData
+        // Pass context data for cache creation
       );
     }
     const metadata = this.afterTaskRunning(executor);
+    if (this.opts.enableCumulativeContext && this.contextStore) {
+      try {
+        const executionResult = this.analyzeExecutionResults(executor, originalPrompt);
+        this.contextStore.addStep({
+          type: "action",
+          summary: `Action: ${processedPrompt}`,
+          prompt: processedPrompt,
+          executionResult
+        });
+        debug4("Added action step with execution result to context store:", {
+          stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
+          totalSteps: this.contextStore.getRecentSteps(100).length,
+          executionResult
+        });
+      } catch (error) {
+        debug4("Failed to analyze execution results, adding step without execution result:", error);
+        try {
+          this.contextStore.addStep({
+            type: "action",
+            summary: `Action: ${processedPrompt}`,
+            prompt: processedPrompt
+          });
+        } catch (stepError) {
+          debug4("Failed to add action step:", stepError);
+        }
+      }
+    }
     return {
       result: output,
       metadata
@@ -2190,38 +2356,50 @@ var PageAgent = class {
       debug4("Context store not available:", error);
     }
     const { output, executor } = await this.taskExecutor.query(processedDemand);
-    if (storageKey && output) {
-      try {
-        const aiModel = await import("misoai-core/ai-model");
-        const contextStore = aiModel.getContextStore();
-        const pendingAliases = contextStore._pendingAliases;
-        if (pendingAliases) {
-          contextStore.storeDataWithAliases(storageKey, output, pendingAliases, typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand));
-          delete contextStore._pendingAliases;
-        } else {
-          contextStore.storeData(storageKey, output);
+    if (this.opts.enableCumulativeContext && this.contextStore) {
+      if (storageKey && output) {
+        try {
+          const pendingAliases = this.contextStore._pendingAliases;
+          if (pendingAliases) {
+            this.contextStore.storeDataWithAliases(storageKey, output, pendingAliases, typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand));
+            delete this.contextStore._pendingAliases;
+            debug4("Stored query result with aliases:", {
+              key: storageKey,
+              value: output,
+              aliases: pendingAliases
+            });
+          } else {
+            this.contextStore.storeData(storageKey, output);
+            debug4("Stored query result:", {
+              key: storageKey,
+              value: output
+            });
+          }
+          this.contextStore.addStep({
+            type: "query",
+            summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
+            data: output,
+            prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
+          });
+          debug4("Added query step to context store:", {
+            storageKey,
+            totalStoredItems: Object.keys(this.contextStore.getAllData()).length,
+            totalSteps: this.contextStore.getRecentSteps(100).length
+          });
+        } catch (error) {
+          debug4("Failed to store query result:", error);
+        }
+      } else {
+        try {
+          this.contextStore.addStep({
+            type: "query",
+            summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
+            data: output,
+            prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
+          });
+        } catch (error) {
+          debug4("Failed to add query step:", error);
         }
-        contextStore.addStep({
-          type: "query",
-          summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
-          data: output,
-          prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
-        });
-      } catch (error) {
-        debug4("Failed to store query result:", error);
-      }
-    } else {
-      try {
-        const aiModel = await import("misoai-core/ai-model");
-        const contextStore = aiModel.getContextStore();
-        contextStore.addStep({
-          type: "query",
-          summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
-          data: output,
-          prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
-        });
-      } catch (error) {
-        debug4("Failed to add query step:", error);
       }
     }
     const metadata = this.afterTaskRunning(executor);
@@ -2333,18 +2511,47 @@ var PageAgent = class {
     };
   }
   async aiAssert(assertion, msg, opt) {
-    let processedAssertion = assertion;
-    try {
-      const aiModel = await import("misoai-core/ai-model");
-      const contextStore = aiModel.getContextStore();
-      processedAssertion = contextStore.replaceAllReferences(assertion, "assertion");
-      contextStore.addStep({
-        type: "assertion",
-        summary: `Assertion: ${processedAssertion}`,
-        prompt: processedAssertion
-      });
-    } catch (error) {
-      debug4("Context store not available:", error);
+    let executionContext = "";
+    if (this.opts.enableCumulativeContext && this.contextStore) {
+      try {
+        const recentSteps = this.contextStore.getRecentSteps(3);
+        const stepsWithExecutionResults = recentSteps.filter((step) => step.executionResult);
+        const storedData = this.contextStore.getAllData();
+        if (stepsWithExecutionResults.length > 0) {
+          const recentActions = stepsWithExecutionResults.map((step) => {
+            const result = step.executionResult;
+            return `- ${result.description}${result.success ? "" : " (FAILED)"}`;
+          }).join("\n");
+          executionContext = `
+
+Recent actions performed:
+${recentActions}
+
+This context may help verify the assertion.`;
+        }
+        if (storedData && Object.keys(storedData).length > 0) {
+          executionContext += `
+
+Available data for reference:
+${JSON.stringify(storedData, null, 2)}
+
+Note: If the assertion references any data keys or natural language equivalents, consider the stored values when verifying.`;
+          debug4("Available data for aiAssert:", {
+            assertion,
+            availableData: storedData
+          });
+        }
+        this.contextStore.addStep({
+          type: "assertion",
+          summary: `Assertion: ${assertion}`,
+          prompt: assertion
+        });
+        debug4("Added assertion step to context store:", {
+          totalSteps: this.contextStore.getRecentSteps(100).length
+        });
+      } catch (error) {
+        debug4("Context store operation failed:", error);
+      }
     }
     let currentUrl = "";
     if (this.page.url) {
@@ -2353,7 +2560,13 @@ var PageAgent = class {
       } catch (e) {
       }
     }
-    const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${processedAssertion}` : processedAssertion;
+    let assertionWithContext = assertion;
+    if (currentUrl) {
+      assertionWithContext = `For the page at URL "${currentUrl}", ${assertion}`;
+    }
+    if (executionContext) {
+      assertionWithContext += executionContext;
+    }
     const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
     const metadata = this.afterTaskRunning(executor, true);
     if (output && opt?.keepRawResponse) {
@@ -2567,6 +2780,79 @@ ${errors}`);
   async destroy() {
     await this.page.destroy();
   }
+  /**
+   * Analyze execution results from executor to generate meaningful descriptions
+   */
+  analyzeExecutionResults(executor, originalPrompt) {
+    const tasks = executor.tasks;
+    const success = !executor.isInErrorState();
+    if (!success) {
+      const errorTask = executor.latestErrorTask();
+      return {
+        success: false,
+        actionType: "error",
+        description: `Failed to execute: ${originalPrompt}`,
+        error: errorTask?.error
+      };
+    }
+    const actionTasks = tasks.filter((t) => t.type === "Action" && t.status === "finished");
+    const locateTasks = tasks.filter((t) => t.type === "Insight" && t.subType === "Locate");
+    const lastAction = actionTasks[actionTasks.length - 1];
+    const lastLocate = locateTasks[locateTasks.length - 1];
+    if (!lastAction) {
+      return {
+        success: true,
+        actionType: "unknown",
+        description: `Completed: ${originalPrompt}`
+      };
+    }
+    const actionType = lastAction.subType || "unknown";
+    const elementInfo = this.extractElementInfo(lastLocate, lastAction);
+    const description = this.generateActionDescription(actionType, lastAction.param, elementInfo);
+    return {
+      success: true,
+      actionType,
+      description,
+      elementInfo,
+      timing: lastAction.timing?.cost
+    };
+  }
+  /**
+   * Extract element information from locate task
+   */
+  extractElementInfo(locateTask, _actionTask) {
+    if (!locateTask?.output?.element)
+      return void 0;
+    const element = locateTask.output.element;
+    return {
+      type: element.attributes?.nodeType || "unknown",
+      text: element.content || element.attributes?.placeholder || element.attributes?.title || "",
+      location: `(${element.center[0]}, ${element.center[1]})`
+    };
+  }
+  /**
+   * Generate natural language description for actions
+   */
+  generateActionDescription(actionType, param, elementInfo) {
+    const elementDesc = elementInfo ? `'${elementInfo.text || elementInfo.type}' element` : "element";
+    switch (actionType) {
+      case "Tap":
+        return `Clicked on ${elementDesc}`;
+      case "Input":
+        const inputValue = param?.value || "";
+        return `Entered "${inputValue}" into ${elementDesc}`;
+      case "KeyboardPress":
+        return `Pressed ${param?.value || "key"}`;
+      case "Scroll":
+        return `Scrolled ${param?.direction || "on page"}`;
+      case "Hover":
+        return `Hovered over ${elementDesc}`;
+      case "Drag":
+        return `Dragged ${elementDesc}`;
+      default:
+        return `Performed ${actionType} action on ${elementDesc}`;
+    }
+  }
 };
 
 // src/playground/agent.ts