misoai-web 1.5.9 → 1.6.1

package/dist/es/midscene-playground.js

@@ -1,4 +1,5 @@
 // src/common/utils.ts
+import dayjs from "dayjs";
 import { elementByPositionWithElementInfo } from "misoai-core/ai-model";
 import { uploadTestInfoToServer } from "misoai-core/utils";
 import { MIDSCENE_REPORT_TAG_NAME, getAIConfig } from "misoai-shared/env";
@@ -10,7 +11,6 @@ import {
 } from "misoai-shared/extractor";
 import { resizeImgBase64 } from "misoai-shared/img";
 import { assert, logMsg, uuid } from "misoai-shared/utils";
-import dayjs from "dayjs";
 
 // src/web-element.ts
 var WebElementInfo = class {
@@ -429,8 +429,8 @@ var ScriptPlayer = class {
 import yaml from "js-yaml";
 
 // src/yaml/utils.ts
-import { assert as assert3 } from "misoai-shared/utils";
 import yaml2 from "js-yaml";
+import { assert as assert3 } from "misoai-shared/utils";
 function interpolateEnvVars(content) {
   return content.replace(/\$\{([^}]+)\}/g, (_, envVar) => {
     const value = process.env[envVar.trim()];
@@ -621,8 +621,12 @@ var WorkflowMemory = class {
     const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
     workflow.memory = [...memory];
     workflow.metadata.totalSteps = workflow.steps.length;
-    workflow.metadata.completedSteps = workflow.steps.filter((s) => s.status === "completed").length;
-    workflow.metadata.failedSteps = workflow.steps.filter((s) => s.status === "failed").length;
+    workflow.metadata.completedSteps = workflow.steps.filter(
+      (s) => s.status === "completed"
+    ).length;
+    workflow.metadata.failedSteps = workflow.steps.filter(
+      (s) => s.status === "failed"
+    ).length;
     this.workflows.set(workflowId, workflow);
     this.enforceRetentionPolicy();
   }
@@ -633,7 +637,9 @@ var WorkflowMemory = class {
     const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
     workflow.context = { ...workflow.context, ...context };
     if (context.currentStep) {
-      const existingStep = workflow.steps.find((s) => s.stepName === context.currentStep);
+      const existingStep = workflow.steps.find(
+        (s) => s.stepName === context.currentStep
+      );
       if (!existingStep) {
         workflow.steps.push({
           stepId: `step_${workflow.steps.length + 1}`,
@@ -678,7 +684,9 @@ var WorkflowMemory = class {
   enforceRetentionPolicy() {
     const maxWorkflows = 10;
     if (this.workflows.size > maxWorkflows) {
-      const sortedWorkflows = Array.from(this.workflows.entries()).sort(([, a], [, b]) => (b.metadata.endTime || b.metadata.startTime) - (a.metadata.endTime || a.metadata.startTime));
+      const sortedWorkflows = Array.from(this.workflows.entries()).sort(
+        ([, a], [, b]) => (b.metadata.endTime || b.metadata.startTime) - (a.metadata.endTime || a.metadata.startTime)
+      );
       const toDelete = sortedWorkflows.slice(maxWorkflows);
       toDelete.forEach(([workflowId]) => this.workflows.delete(workflowId));
     }
@@ -1417,7 +1425,9 @@ var PageTaskExecutor = class {
    */
   getPersistentExecutor() {
     if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
-      const previousMemory = this.workflowMemory.getWorkflowMemory(this.sessionContext.workflowId);
+      const previousMemory = this.workflowMemory.getWorkflowMemory(
+        this.sessionContext.workflowId
+      );
       this.persistentExecutor = new Executor("Persistent Task Executor", {
         onTaskStart: this.onTaskStartCallback,
         initialMemory: previousMemory
@@ -1446,7 +1456,9 @@ var PageTaskExecutor = class {
     if (this.persistentExecutor) {
       this.persistentExecutor.clearMemory();
     }
-    this.workflowMemory.clearWorkflow(this.sessionContext.workflowId || "default");
+    this.workflowMemory.clearWorkflow(
+      this.sessionContext.workflowId || "default"
+    );
   }
   /**
    * Mevcut hafızayı döndürür
@@ -1458,7 +1470,9 @@ var PageTaskExecutor = class {
    * İş akışı hafızasını döndürür
    */
   getWorkflowMemory() {
-    return this.workflowMemory.getWorkflowData(this.sessionContext.workflowId || "default");
+    return this.workflowMemory.getWorkflowData(
+      this.sessionContext.workflowId || "default"
+    );
   }
   /**
    * Hafıza istatistiklerini döndürür
@@ -1466,7 +1480,13 @@ var PageTaskExecutor = class {
   getMemoryStats() {
     return this.persistentExecutor?.getMemoryStats() || {
       totalItems: 0,
-      analytics: { totalTasks: 0, memoryHits: 0, memoryMisses: 0, averageMemorySize: 0, memoryEffectiveness: 0 },
+      analytics: {
+        totalTasks: 0,
+        memoryHits: 0,
+        memoryMisses: 0,
+        averageMemorySize: 0,
+        memoryEffectiveness: 0
+      },
       config: this.memoryConfig
     };
   }
@@ -1502,11 +1522,14 @@ var PageTaskExecutor = class {
     let taskExecutor;
     if (useMemory) {
       taskExecutor = this.getPersistentExecutor();
-      this.workflowMemory.updateWorkflowContext({
-        currentStep: title,
-        pageInfo: this.sessionContext.pageInfo,
-        timestamp: Date.now()
-      }, this.sessionContext.workflowId || "default");
+      this.workflowMemory.updateWorkflowContext(
+        {
+          currentStep: title,
+          pageInfo: this.sessionContext.pageInfo,
+          timestamp: Date.now()
+        },
+        this.sessionContext.workflowId || "default"
+      );
     } else {
       taskExecutor = new Executor(title, {
         onTaskStart: this.onTaskStartCallback
@@ -1804,9 +1827,15 @@ var PageTaskExecutor = class {
   }
   async waitFor(assertion, opt) {
     const description = `waitFor: ${assertion}`;
-    const taskExecutor = new Executor(taskTitleStr("WaitFor", description), {
-      onTaskStart: this.onTaskStartCallback
-    });
+    const useMemory = true;
+    let taskExecutor;
+    if (useMemory) {
+      taskExecutor = this.getPersistentExecutor();
+    } else {
+      taskExecutor = new Executor(taskTitleStr("WaitFor", description), {
+        onTaskStart: this.onTaskStartCallback
+      });
+    }
     const { timeoutMs, checkIntervalMs } = opt;
     assert4(assertion, "No assertion for waitFor");
     assert4(timeoutMs, "No timeoutMs for waitFor");
@@ -1861,6 +1890,25 @@ var PageTaskExecutor = class {
       `waitFor timeout: ${errorThought}`
     );
   }
+  /**
+   * Hafızaya yeni bir öğe ekler
+   */
+  addToMemory(memoryItem) {
+    if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
+      const previousMemory = this.workflowMemory.getWorkflowMemory(
+        this.sessionContext.workflowId
+      );
+      this.persistentExecutor = new Executor("Persistent Task Executor", {
+        onTaskStart: this.onTaskStartCallback,
+        initialMemory: previousMemory
+      });
+    }
+    this.persistentExecutor.memoryStore?.add(memoryItem);
+    this.persistentExecutor.memoryAnalytics?.recordMemoryOperation(
+      "add",
+      memoryItem
+    );
+  }
 };
 
 // src/common/plan-builder.ts
@@ -1948,14 +1996,14 @@ function buildPlans(type, locateParam, param) {
 import assert6 from "assert";
 import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync, writeFileSync as writeFileSync2 } from "fs";
 import { dirname as dirname2, join as join2 } from "path";
+import yaml3 from "js-yaml";
 import { getMidsceneRunSubDir as getMidsceneRunSubDir2 } from "misoai-shared/common";
 import { getDebug as getDebug3 } from "misoai-shared/logger";
 import { ifInBrowser as ifInBrowser2 } from "misoai-shared/utils";
-import yaml3 from "js-yaml";
 import semver from "semver";
 
 // package.json
-var version = "1.5.8";
+var version = "1.6.1";
 
 // src/common/task-cache.ts
 var debug3 = getDebug3("cache");
@@ -2166,7 +2214,10 @@ var PageAgent = class {
     }
     this.taskExecutor = new PageTaskExecutor(this.page, this.insight, {
       taskCache: this.taskCache,
-      onTaskStart: this.callbackOnTaskStartTip.bind(this)
+      onTaskStart: this.callbackOnTaskStartTip.bind(this),
+      memoryConfig: opts?.memoryConfig,
+      sessionId: opts?.sessionId,
+      workflowId: opts?.workflowId
     });
     this.dump = this.resetDump();
     this.reportFileName = reportFileName(
@@ -2247,22 +2298,28 @@ var PageAgent = class {
     const allThoughts = executor.tasks.filter((task) => task.thought).map((task) => task.thought);
     const allLocates = executor.tasks.filter((task) => task.locate).map((task) => task.locate);
     const allPlans = executor.tasks.filter((task) => task.param?.plans).map((task) => task.param?.plans);
-    const planningTasks = executor.tasks.filter((task) => task.type === "Planning");
-    const insightTasks = executor.tasks.filter((task) => task.type === "Insight");
+    const planningTasks = executor.tasks.filter(
+      (task) => task.type === "Planning"
+    );
+    const insightTasks = executor.tasks.filter(
+      (task) => task.type === "Insight"
+    );
     const actionTasks = executor.tasks.filter((task) => task.type === "Action");
     const planning = planningTasks.length > 0 ? {
       type: "Planning",
-      description: `Planning for task execution`,
+      description: "Planning for task execution",
       steps: planningTasks.map((task) => task.thought || "Planning step")
     } : void 0;
     const insight = insightTasks.length > 0 ? {
       type: "Insight",
-      description: `Insight for task execution`,
-      elements: insightTasks.map((task) => task.thought || "Insight element")
+      description: "Insight for task execution",
+      elements: insightTasks.map(
+        (task) => task.thought || "Insight element"
+      )
     } : void 0;
     const action = actionTasks.length > 0 ? {
       type: "Action",
-      description: `Action for task execution`,
+      description: "Action for task execution",
       result: lastTask?.output
     } : void 0;
     const actionDetails = executor.tasks.map((task) => ({
@@ -2596,7 +2653,10 @@ ${memoryContext}` : void 0;
     }
     const memoryContext = this.getMemoryAsContext();
     const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
-    const { output, executor } = await this.taskExecutor.assert(assertionWithContext, memoryContext);
+    const { output, executor } = await this.taskExecutor.assert(
+      assertionWithContext,
+      memoryContext
+    );
     const metadata = this.afterTaskRunning(executor, true);
     if (output && opt?.keepRawResponse) {
       return {
@@ -2617,6 +2677,7 @@ ${reasonMsg}`);
   }
   async aiCaptcha(options) {
     const { deepThink = false, autoDetectComplexity = true } = options || {};
+    const memoryContext = this.getMemoryAsContext();
     let shouldUseDeepThink = deepThink;
     if (autoDetectComplexity && !deepThink) {
       const context = await this.getUIContext();
@@ -2634,7 +2695,10 @@ A complex CAPTCHA typically has one or more of these characteristics:
 Return only "complex" or "simple" based on your analysis.
 `;
         const complexityMsgs = [
-          { role: "system", content: "You are an AI assistant that analyzes screenshots to determine CAPTCHA complexity." },
+          {
+            role: "system",
+            content: "You are an AI assistant that analyzes screenshots to determine CAPTCHA complexity."
+          },
           {
             role: "user",
             content: [
@@ -2658,7 +2722,12 @@ Return only "complex" or "simple" based on your analysis.
         );
         const responseText = typeof complexityResult.content === "string" ? complexityResult.content.toLowerCase() : JSON.stringify(complexityResult.content).toLowerCase();
         shouldUseDeepThink = responseText.includes("complex");
-        debug4("CAPTCHA complexity analysis:", responseText, "Using deep think:", shouldUseDeepThink);
+        debug4(
+          "CAPTCHA complexity analysis:",
+          responseText,
+          "Using deep think:",
+          shouldUseDeepThink
+        );
       } catch (error) {
         debug4("Failed to analyze CAPTCHA complexity:", error);
       }
@@ -2675,7 +2744,9 @@ Return only "complex" or "simple" based on your analysis.
           await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
         } else if (action.type === "input" && action.value) {
           if (action.target) {
-            await this.aiInput(action.value, action.target, { deepThink: shouldUseDeepThink });
+            await this.aiInput(action.value, action.target, {
+              deepThink: shouldUseDeepThink
+            });
           }
         } else if (action.type === "verify" && action.target) {
           await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
@@ -2687,7 +2758,9 @@ Return only "complex" or "simple" based on your analysis.
           if (action.coordinates) {
             const x = action.coordinates[0];
             const y = action.coordinates[1];
-            await this.aiTap(`element at coordinates (${x}, ${y})`, { deepThink: shouldUseDeepThink });
+            await this.aiTap(`element at coordinates (${x}, ${y})`, {
+              deepThink: shouldUseDeepThink
+            });
           } else if (action.target) {
             await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
           }
@@ -2697,6 +2770,26 @@ Return only "complex" or "simple" based on your analysis.
       }
     }
     await new Promise((resolve2) => setTimeout(resolve2, 3e3));
+    const captchaMemoryItem = {
+      id: `captcha_${Date.now()}`,
+      timestamp: Date.now(),
+      taskType: "Action",
+      summary: `Solved ${captchaResult.captchaType} CAPTCHA: ${captchaResult.thought}`,
+      context: {
+        url: await this.page.url?.() || "",
+        captchaType: captchaResult.captchaType,
+        actions: captchaResult.actions,
+        deepThink: actualDeepThink
+      },
+      metadata: {
+        executionTime: Date.now() - Date.now(),
+        // Will be updated
+        success: true,
+        confidence: 0.9
+      },
+      tags: ["captcha", "action", captchaResult.captchaType]
+    };
+    this.taskExecutor.addToMemory(captchaMemoryItem);
     const metadata = {
       status: "finished",
       usage,
@@ -2713,10 +2806,15 @@ Return only "complex" or "simple" based on your analysis.
   }
   async aiWaitFor(assertion, opt) {
     const startTime = Date.now();
-    const { executor } = await this.taskExecutor.waitFor(assertion, {
+    const memoryContext = this.getMemoryAsContext();
+    const assertionWithContext = memoryContext ? `${assertion}
+
+Previous workflow steps:
+${memoryContext}` : assertion;
+    const { executor } = await this.taskExecutor.waitFor(assertionWithContext, {
       timeoutMs: opt?.timeoutMs || 15 * 1e3,
       checkIntervalMs: opt?.checkIntervalMs || 3 * 1e3,
-      assertion
+      assertion: assertionWithContext
     });
     const metadata = {
       status: executor.isInErrorState() ? "failed" : "finished",
@@ -2814,25 +2912,27 @@ ${errors}`);
       const executionDump = {
         name: screenshotTitle,
         description: content,
-        tasks: [{
-          type: "Screenshot",
-          subType: "log",
-          status: "finished",
-          executor: null,
-          param: {
-            title: screenshotTitle,
-            content
-          },
-          output: {
-            screenshot
-          },
-          thought: `Logged screenshot: ${screenshotTitle}`,
-          timing: {
-            start: Date.now(),
-            end: Date.now(),
-            cost: 0
+        tasks: [
+          {
+            type: "Screenshot",
+            subType: "log",
+            status: "finished",
+            executor: null,
+            param: {
+              title: screenshotTitle,
+              content
+            },
+            output: {
+              screenshot
+            },
+            thought: `Logged screenshot: ${screenshotTitle}`,
+            timing: {
+              start: Date.now(),
+              end: Date.now(),
+              cost: 0
+            }
           }
-        }],
+        ],
         sdkVersion: "1.0.0",
         logTime: Date.now(),
         model_name: "screenshot"
@@ -2884,7 +2984,9 @@ ${errors}`);
         totalTasks: stats.analytics.totalTasks,
         memoryHits: stats.analytics.memoryHits,
         memoryMisses: stats.analytics.memoryMisses,
-        memoryEffectiveness: Math.round(stats.analytics.memoryEffectiveness * 100),
+        memoryEffectiveness: Math.round(
+          stats.analytics.memoryEffectiveness * 100
+        ),
         averageMemorySize: Math.round(stats.analytics.averageMemorySize * 100) / 100
       },
       config: stats.config,
@@ -2948,7 +3050,9 @@ ${errors}`);
   calculateSuccessRate(memory) {
     if (memory.length === 0)
       return 0;
-    const successCount = memory.filter((item) => item.metadata?.success !== false).length;
+    const successCount = memory.filter(
+      (item) => item.metadata?.success !== false
+    ).length;
     return Math.round(successCount / memory.length * 100);
   }
   calculateAverageExecutionTime(memory) {
@@ -3085,13 +3189,13 @@ var StaticPage = class {
 import { randomUUID } from "crypto";
 import { existsSync as existsSync3, readFileSync as readFileSync2, writeFileSync as writeFileSync3 } from "fs";
 import { join as join3 } from "path";
+import cors from "cors";
+import dotenv from "dotenv";
+import express from "express";
 import { getTmpDir } from "misoai-core/utils";
 import { PLAYGROUND_SERVER_PORT } from "misoai-shared/constants";
 import { overrideAIConfig } from "misoai-shared/env";
 import { ifInBrowser as ifInBrowser3 } from "misoai-shared/utils";
-import cors from "cors";
-import dotenv from "dotenv";
-import express from "express";
 var defaultPort = PLAYGROUND_SERVER_PORT;
 var errorHandler = (err, req, res, next) => {
   console.error(err);