@donggui/core 1.5.14 → 1.6.0

package/dist/es/agent/agent.mjs

@@ -1,3 +1,4 @@
+import { AiAssertElement } from "../ai-model/assert.mjs";
 import { ScreenshotItem } from "../screenshot-item.mjs";
 import service from "../service/index.mjs";
 import { ExecutionDump, GroupedActionDump } from "../types.mjs";
@@ -7,7 +8,7 @@ import { ReportGenerator } from "../report-generator.mjs";
 import { getVersion, processCacheConfig, reportHTMLContent } from "../utils.mjs";
 import { ScriptPlayer, buildDetailedLocateParam, parseYamlScript } from "../yaml/index.mjs";
 import { existsSync } from "node:fs";
-import { resolve } from "node:path";
+import { resolve as external_node_path_resolve } from "node:path";
 import { MIDSCENE_REPLANNING_CYCLE_LIMIT, ModelConfigManager, globalConfigManager, globalModelConfigManager } from "@midscene/shared/env";
 import { getDebug } from "@midscene/shared/logger";
 import { assert, ifInBrowser, uuid } from "@midscene/shared/utils";
@@ -162,6 +163,7 @@ class Agent {
     }
     async callActionInActionSpace(type, opt) {
         debug('callActionInActionSpace', type, ',', opt);
+        const beforeScreenshot = await this.interface.screenshotBase64();
         const actionPlan = {
             type: type,
             param: opt || {},
@@ -175,6 +177,15 @@ class Agent {
         const defaultIntentModelConfig = this.modelConfigManager.getModelConfig('default');
         const modelConfigForPlanning = this.modelConfigManager.getModelConfig('planning');
         const { output } = await this.taskExecutor.runPlans(title, plans, modelConfigForPlanning, defaultIntentModelConfig);
+        const afterScreenshot = await this.interface.screenshotBase64();
+        this.actionScreenshotHistory.push({
+            beforeScreenshot,
+            afterScreenshot,
+            actionType: type,
+            actionParam: opt,
+            timestamp: Date.now()
+        });
+        if (this.actionScreenshotHistory.length > this.maxScreenshotHistoryLength) this.actionScreenshotHistory.shift();
         return output;
     }
     async aiTap(locatePrompt, opt) {
@@ -290,6 +301,8 @@ class Agent {
         const abortSignal = opt?.abortSignal;
         if (abortSignal?.aborted) throw new Error(`aiAct aborted: ${abortSignal.reason || 'signal already aborted'}`);
         const runAiAct = async ()=>{
+            this.aiActStartScreenshot = await this.interface.screenshotBase64();
+            this.actionScreenshotHistory = [];
             const modelConfigForPlanning = this.modelConfigManager.getModelConfig('planning');
             const defaultIntentModelConfig = this.modelConfigManager.getModelConfig('default');
             const deepThink = opt?.deepThink === 'unset' ? void 0 : opt?.deepThink;
@@ -426,19 +439,169 @@ class Agent {
     }
     async aiAssert(assertion, msg, opt) {
         const modelConfig = this.modelConfigManager.getModelConfig('insight');
-        const serviceOpt = {
-            domIncluded: opt?.domIncluded ?? defaultServiceExtractOption.domIncluded,
-            screenshotIncluded: opt?.screenshotIncluded ?? defaultServiceExtractOption.screenshotIncluded
-        };
-        const { textPrompt, multimodalPrompt } = parsePrompt(assertion);
+        const { textPrompt } = parsePrompt(assertion);
         const assertionText = 'string' == typeof assertion ? assertion : assertion.prompt;
+        let beforeScreenshot;
+        let afterScreenshot;
+        const mode = opt?.screenshotMode || 'auto';
+        switch(mode){
+            case 'manual':
+                beforeScreenshot = opt?.beforeScreenshot;
+                afterScreenshot = opt?.afterScreenshot;
+                break;
+            case 'flow':
+                beforeScreenshot = this.aiActStartScreenshot || this.actionScreenshotHistory[0]?.beforeScreenshot;
+                afterScreenshot = await this.interface.screenshotBase64();
+                break;
+            case 'currentOnly':
+                beforeScreenshot = void 0;
+                afterScreenshot = await this.interface.screenshotBase64();
+                break;
+            case 'lastAction':
+                if (this.actionScreenshotHistory.length > 0) {
+                    const lastContext = this.actionScreenshotHistory[this.actionScreenshotHistory.length - 1];
+                    beforeScreenshot = lastContext.beforeScreenshot;
+                    afterScreenshot = await this.interface.screenshotBase64();
+                } else {
+                    beforeScreenshot = void 0;
+                    afterScreenshot = await this.interface.screenshotBase64();
+                }
+                break;
+            case 'diff':
+                afterScreenshot = await this.interface.screenshotBase64();
+                break;
+            case 'video':
+                beforeScreenshot = void 0;
+                afterScreenshot = await this.interface.screenshotBase64();
+                break;
+            default:
+                if (opt?.beforeScreenshot && opt?.afterScreenshot) {
+                    beforeScreenshot = opt.beforeScreenshot;
+                    afterScreenshot = opt.afterScreenshot;
+                } else if (this.actionScreenshotHistory.length > 0) {
+                    const lastContext = this.actionScreenshotHistory[this.actionScreenshotHistory.length - 1];
+                    beforeScreenshot = lastContext.beforeScreenshot;
+                    afterScreenshot = await this.interface.screenshotBase64();
+                } else {
+                    beforeScreenshot = void 0;
+                    afterScreenshot = await this.interface.screenshotBase64();
+                }
+                break;
+        }
+        const executeAssertion = async ()=>{
+            if ('diff' === mode && opt?.referenceImages && opt.referenceImages.length > 0) {
+                const { AiAssertDiff } = await import("../ai-model/assert.mjs");
+                const diffResult = await AiAssertDiff({
+                    currentScreenshot: afterScreenshot,
+                    referenceImages: opt.referenceImages,
+                    assertion: textPrompt,
+                    businessContext: opt.businessContext,
+                    diffThreshold: opt.diffThreshold,
+                    ignoreRegions: opt.ignoreRegions,
+                    ignoreDynamicContent: opt.ignoreDynamicContent,
+                    strictMode: opt.strictMode,
+                    modelConfig
+                });
+                return {
+                    pass: diffResult.pass,
+                    thought: diffResult.thought,
+                    reason: diffResult.reason,
+                    systemCheckResults: void 0,
+                    diffDetails: diffResult.diffDetails,
+                    videoDetails: void 0
+                };
+            }
+            if ('video' === mode && opt?.currentVideo) {
+                const { AiAssertVideo } = await import("../ai-model/assert.mjs");
+                const MAX_DURATION = 5;
+                const DEFAULT_FPS = 30;
+                const MAX_FRAMES = MAX_DURATION * DEFAULT_FPS;
+                let videoFrames = [];
+                if (opt.currentVideo.frames && opt.currentVideo.frames.length > 0) videoFrames = opt.currentVideo.frames.slice(0, MAX_FRAMES);
+                else if ('url' === opt.currentVideo.format && opt.currentVideo.url) throw new Error('Video URL format is not yet supported. Please provide video frames directly.');
+                else throw new Error('currentVideo.frames is required for video assertion mode');
+                if (0 === videoFrames.length) throw new Error('No video frames provided for video assertion');
+                const videoResult = await AiAssertVideo({
+                    currentVideoFrames: videoFrames,
+                    assertion: textPrompt,
+                    businessContext: opt.businessContext,
+                    videoOptions: opt.videoOptions,
+                    modelConfig
+                });
+                return {
+                    pass: videoResult.pass,
+                    thought: videoResult.thought,
+                    reason: videoResult.reason,
+                    systemCheckResults: void 0,
+                    diffDetails: void 0,
+                    videoDetails: videoResult.videoDetails
+                };
+            }
+            const assertResult = await AiAssertElement({
+                beforeScreenshot,
+                afterScreenshot,
+                assertion: textPrompt,
+                businessContext: opt?.businessContext,
+                enableSystemCheck: opt?.enableSystemCheck ?? true,
+                customSystemCheckRules: opt?.customSystemCheckRules,
+                modelConfig
+            });
+            return {
+                pass: assertResult.pass,
+                thought: assertResult.thought,
+                reason: assertResult.reason,
+                systemCheckResults: assertResult.systemCheckResults,
+                diffDetails: void 0
+            };
+        };
+        const runWithRetry = async ()=>{
+            const retryOptions = opt?.retryOptions;
+            let lastError;
+            let attempts = 0;
+            const maxAttempts = retryOptions?.maxRetries ? retryOptions.maxRetries + 1 : 1;
+            while(attempts < maxAttempts)try {
+                attempts++;
+                const result = await executeAssertion();
+                return {
+                    pass: result.pass,
+                    thought: result.thought,
+                    reason: result.reason,
+                    systemCheckResults: result.systemCheckResults,
+                    diffDetails: result.diffDetails
+                };
+            } catch (error) {
+                lastError = error instanceof Error ? error : new Error(String(error));
+                if (attempts < maxAttempts && retryOptions?.retryInterval) await new Promise((resolve)=>setTimeout(resolve, retryOptions.retryInterval));
+            }
+            throw lastError || new Error('Assertion failed after retries');
+        };
         try {
-            const { output, thought } = await this.taskExecutor.createTypeQueryExecution('Assert', textPrompt, modelConfig, serviceOpt, multimodalPrompt);
-            const pass = Boolean(output);
-            const message = pass ? void 0 : `Assertion failed: ${msg || assertionText}\nReason: ${thought || '(no_reason)'}`;
+            const result = await runWithRetry();
+            const { pass, thought, reason, systemCheckResults, diffDetails } = result;
+            if (opt?.saveSnapshot) {
+                const snapshot = {
+                    beforeScreenshot,
+                    afterScreenshot,
+                    timestamp: Date.now(),
+                    assertion: assertionText
+                };
+                if (opt?.snapshotPath) {
+                    const fs = await import("node:fs");
+                    const path = await import("node:path");
+                    const snapshotDir = path.dirname(opt.snapshotPath);
+                    if (!fs.existsSync(snapshotDir)) fs.mkdirSync(snapshotDir, {
+                        recursive: true
+                    });
+                    fs.writeFileSync(opt.snapshotPath, JSON.stringify(snapshot, null, 2));
+                }
+            }
+            const message = pass ? void 0 : `Assertion failed: ${msg || assertionText}\nReason: ${reason || thought || '(no_reason)'}`;
             if (opt?.keepRawResponse) return {
                 pass,
                 thought,
+                reason,
+                systemCheckResults,
+                diffDetails,
                 message
             };
             if (!pass) throw new Error(message);
@@ -453,6 +616,7 @@ class Agent {
                 if (opt?.keepRawResponse) return {
                     pass: false,
                     thought,
+                    reason,
                     message
                 };
                 throw new Error(message, {
@@ -462,6 +626,16 @@ class Agent {
             throw error;
         }
     }
+    clearActionScreenshotHistory() {
+        this.actionScreenshotHistory = [];
+        this.aiActStartScreenshot = null;
+    }
+    getLastActionScreenshotContext() {
+        return this.actionScreenshotHistory[this.actionScreenshotHistory.length - 1];
+    }
+    getFlowStartScreenshot() {
+        return this.aiActStartScreenshot;
+    }
     async aiWaitFor(assertion, opt) {
         const modelConfig = this.modelConfigManager.getModelConfig('insight');
         await this.taskExecutor.waitFor(assertion, {
@@ -610,7 +784,7 @@ class Agent {
     normalizeFilePaths(files) {
         if (ifInBrowser) throw new Error('File chooser is not supported in browser environment');
         return files.map((file)=>{
-            const absolutePath = resolve(file);
+            const absolutePath = external_node_path_resolve(file);
             if (!existsSync(absolutePath)) throw new Error(`File not found: ${file}`);
             return absolutePath;
         });
@@ -644,6 +818,9 @@ class Agent {
         _define_property(this, "executionDumpIndexByRunner", new WeakMap());
         _define_property(this, "fullActionSpace", void 0);
         _define_property(this, "reportGenerator", void 0);
+        _define_property(this, "actionScreenshotHistory", []);
+        _define_property(this, "maxScreenshotHistoryLength", 5);
+        _define_property(this, "aiActStartScreenshot", null);
         this.interface = interfaceInstance;
         const envReplanningCycleLimit = globalConfigManager.getEnvConfigValueAsNumber(MIDSCENE_REPLANNING_CYCLE_LIMIT);
         this.opts = Object.assign({