@donggui/core 1.5.14 → 1.6.1

package/dist/types/agent/agent.d.ts

@@ -1,6 +1,6 @@
 import type { TUserPrompt } from '../ai-model/index';
 import Service from '../service/index';
-import { type ActionParam, type ActionReturn, type AgentAssertOpt, type AgentDescribeElementAtPointResult, type AgentOpt, type AgentWaitForOpt, type DeepThinkOption, type DeviceAction, ExecutionDump, GroupedActionDump, type LocateOption, type LocateResultElement, type LocateValidatorResult, type LocatorValidatorOption, type OnTaskStartTip, type ScrollParam, type ServiceAction, type ServiceExtractOption, type ServiceExtractParam, type UIContext } from '../types';
+import { type ActionParam, type ActionReturn, type ActionScreenshotContext, type AgentAssertOpt, type AgentDescribeElementAtPointResult, type AgentOpt, type AgentWaitForOpt, type DeepThinkOption, type DeviceAction, ExecutionDump, GroupedActionDump, type LocateOption, type LocateResultElement, type LocateValidatorResult, type LocatorValidatorOption, type OnTaskStartTip, type ScrollParam, type ServiceAction, type ServiceExtractOption, type ServiceExtractParam, type UIContext } from '../types';
 export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted';
 import type { AbstractInterface } from '../device';
 import type { TaskRunner } from '../task-runner';
@@ -45,6 +45,19 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
     private executionDumpIndexByRunner;
     private fullActionSpace;
     private reportGenerator;
+    /**
+     * Action screenshot context history for assertion
+     * Stores the last N action screenshots for comparison in assertions
+     */
+    private actionScreenshotHistory;
+    /**
+     * Maximum number of action screenshots to keep in history
+     */
+    private readonly maxScreenshotHistoryLength;
+    /**
+     * Screenshot at the start of aiAct for flow-level assertion
+     */
+    private aiActStartScreenshot;
     get page(): InterfaceType;
     /**
      * Ensures VL model warning is shown once when needed
@@ -124,9 +137,32 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
     aiLocate(prompt: TUserPrompt, opt?: LocateOption): Promise<Pick<LocateResultElement, "rect" | "center">>;
     aiAssert(assertion: TUserPrompt, msg?: string, opt?: AgentAssertOpt & ServiceExtractOption): Promise<{
         pass: boolean;
-        thought: string | undefined;
+        thought: string;
+        reason: string | undefined;
+        systemCheckResults: any;
+        diffDetails: any;
         message: string | undefined;
+    } | {
+        pass: boolean;
+        thought: string | undefined;
+        reason: string;
+        message: string;
+        systemCheckResults?: undefined;
+        diffDetails?: undefined;
     } | undefined>;
+    /**
+     * Clear the action screenshot history
+     * Useful when starting a new test scenario
+     */
+    clearActionScreenshotHistory(): void;
+    /**
+     * Get the last action screenshot context
+     */
+    getLastActionScreenshotContext(): ActionScreenshotContext | undefined;
+    /**
+     * Get the flow start screenshot (from aiAct)
+     */
+    getFlowStartScreenshot(): string | null;
     aiWaitFor(assertion: TUserPrompt, opt?: AgentWaitForOpt): Promise<void>;
     ai(...args: Parameters<typeof this.aiAct>): Promise<string | undefined>;
     runYaml(yamlScriptContent: string): Promise<{

package/dist/types/ai-model/assert.d.ts

@@ -0,0 +1,66 @@
+import type { AIUsageInfo, DiffDetails, IgnoreRegion, ReferenceImage, SystemCheckResults, VideoAssertionOptions, VideoDetails } from '../types';
+import type { IModelConfig } from '@midscene/shared/env';
+export interface AiAssertOptions {
+    beforeScreenshot?: string;
+    afterScreenshot: string;
+    assertion: string;
+    businessContext?: string;
+    enableSystemCheck?: boolean;
+    customSystemCheckRules?: string;
+    modelConfig: IModelConfig;
+    abortSignal?: AbortSignal;
+    referenceImages?: ReferenceImage[];
+    diffThreshold?: number;
+    ignoreRegions?: IgnoreRegion[];
+    ignoreDynamicContent?: boolean;
+    strictMode?: boolean;
+}
+export interface AIAssertionResponse {
+    pass: boolean;
+    thought: string;
+    reason?: string;
+    systemCheckResults?: SystemCheckResults;
+    diffDetails?: DiffDetails;
+}
+export declare function AiAssertElement(options: AiAssertOptions): Promise<{
+    pass: boolean;
+    thought: string;
+    reason?: string;
+    usage?: AIUsageInfo;
+    systemCheckResults?: SystemCheckResults;
+    rawResponse?: string;
+}>;
+export declare function AiAssertDiff(options: {
+    currentScreenshot: string;
+    referenceImages: ReferenceImage[];
+    assertion: string;
+    businessContext?: string;
+    diffThreshold?: number;
+    ignoreRegions?: IgnoreRegion[];
+    ignoreDynamicContent?: boolean;
+    strictMode?: boolean;
+    modelConfig: IModelConfig;
+    abortSignal?: AbortSignal;
+}): Promise<{
+    pass: boolean;
+    thought: string;
+    reason?: string;
+    diffDetails?: DiffDetails;
+    usage?: AIUsageInfo;
+    rawResponse?: string;
+}>;
+export declare function AiAssertVideo(options: {
+    currentVideoFrames: string[];
+    assertion: string;
+    businessContext?: string;
+    videoOptions?: VideoAssertionOptions;
+    modelConfig: IModelConfig;
+    abortSignal?: AbortSignal;
+}): Promise<{
+    pass: boolean;
+    thought: string;
+    reason?: string;
+    videoDetails?: VideoDetails;
+    usage?: AIUsageInfo;
+    rawResponse?: string;
+}>;

package/dist/types/ai-model/index.d.ts

@@ -4,6 +4,8 @@ export { generatePlaywrightTest, generatePlaywrightTestStream, } from './prompt/
 export { generateYamlTest, generateYamlTestStream, } from './prompt/yaml-generator';
 export type { ChatCompletionMessageParam } from 'openai/resources/index';
 export { AiLocateElement, AiExtractElementInfo, AiLocateSection, AiJudgeOrderSensitive, } from './inspect';
+export { AiAssertElement, AiAssertDiff } from './assert';
+export type { AiAssertOptions, AIAssertionResponse } from './assert';
 export { plan } from './llm-planning';
 export { autoGLMPlanning } from './auto-glm/planning';
 export { adaptBboxToRect } from '../common';

package/dist/types/ai-model/prompt/cache-verify.d.ts

@@ -0,0 +1 @@
+export declare const cacheVerifyInstruction: () => string;

package/dist/types/service/index.d.ts

@@ -1,6 +1,6 @@
-import type { AIDescribeElementResponse, DetailedLocateParam, LocateResultWithDump, Rect, ServiceExtractOption, ServiceExtractParam, ServiceExtractResult, ServiceTaskInfo, UIContext } from '../types';
+import type { AIDescribeElementResponse, CacheVerificationResult, DetailedLocateParam, LocateResultWithDump, Rect, ServiceExtractOption, ServiceExtractParam, ServiceExtractResult, ServiceTaskInfo, UIContext } from '../types';
 import type { IModelConfig } from '@midscene/shared/env';
-import type { TMultimodalPrompt } from '../common';
+import type { TMultimodalPrompt, TUserPrompt } from '../common';
 export interface LocateOpts {
     context?: UIContext;
 }
@@ -19,5 +19,6 @@ export default class Service {
     describe(target: Rect | [number, number], modelConfig: IModelConfig, opt?: {
         deepLocate?: boolean;
     }): Promise<Pick<AIDescribeElementResponse, 'description'>>;
+    verifyCachedElement(center: [number, number], targetPrompt: TUserPrompt, modelConfig: IModelConfig, uiContext?: UIContext): Promise<CacheVerificationResult>;
 }
 export {};

package/dist/types/types.d.ts

@@ -60,6 +60,16 @@ export interface LocateValidatorResult {
     center: [number, number];
     centerDistance?: number;
 }
+export interface CacheVerificationResult {
+    pass: boolean;
+    description?: string;
+    reason?: string;
+}
+export interface CacheValidationOptions {
+    enableCoordOffsetCheck?: boolean;
+    coordOffsetThresholdPx?: number;
+    enableVisualVerify?: boolean;
+}
 export interface AgentDescribeElementAtPointResult {
     prompt: string;
     deepLocate: boolean;
@@ -135,6 +145,9 @@ export interface ServiceDump extends DumpMeta {
     data: any;
     assertionPass?: boolean;
     assertionThought?: string;
+    assertionReason?: string;
+    beforeScreenshot?: string;
+    systemCheckResults?: SystemCheckResults;
     taskInfo: ServiceTaskInfo;
     error?: string;
     output?: any;
@@ -172,8 +185,161 @@ export interface AgentWaitForOpt extends ServiceExtractOption {
     checkIntervalMs?: number;
     timeoutMs?: number;
 }
+export interface ReferenceImage {
+    name: string;
+    url: string;
+}
+export interface IgnoreRegion {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}
+export interface VideoInput {
+    /** 视频帧列表（Base64 编码的图片），最多 150 帧（5秒 * 30fps） */
+    frames: string[];
+    /** 视频时长（秒），最大 5 秒 */
+    duration?: number;
+    /** 帧率，默认 30 */
+    fps?: number;
+    /** 视频格式，默认 'frames' */
+    format?: 'frames' | 'base64' | 'url';
+    /** 视频 URL（当 format 为 'url' 时使用） */
+    url?: string;
+}
+export interface VideoAssertionOptions {
+    /** 是否验证动画流畅度，默认 true */
+    checkSmoothness?: boolean;
+    /** 流畅度阈值（0-100），默认 60 */
+    smoothnessThreshold?: number;
+    /** 是否验证动画时长，默认 false */
+    checkDuration?: boolean;
+    /** 预期动画时长范围（秒） */
+    expectedDuration?: {
+        min?: number;
+        max?: number;
+    };
+    /** 是否验证动画完整性，默认 true */
+    checkCompleteness?: boolean;
+    /** 关键帧验证配置 */
+    keyframes?: {
+        /** 关键帧时间点（秒） */
+        timestamps: number[];
+        /** 关键帧描述 */
+        descriptions?: string[];
+    };
+    /** 是否忽略背景变化，默认 false */
+    ignoreBackgroundChanges?: boolean;
+    /** 动画类型，用于优化分析 */
+    animationType?: 'transition' | 'loading' | 'interaction' | 'auto';
+}
+export interface VideoDetails {
+    /** 动画流畅度评分（0-100） */
+    smoothnessScore: number;
+    /** 动画时长（秒） */
+    duration: number;
+    /** 动画是否完整 */
+    isComplete: boolean;
+    /** 关键帧匹配结果 */
+    keyframeMatches?: {
+        timestamp: number;
+        matched: boolean;
+        description?: string;
+    }[];
+    /** 动画类型识别 */
+    detectedAnimationType: string;
+    /** 动画质量评估 */
+    qualityAssessment: {
+        /** 是否有卡顿 */
+        hasStuttering: boolean;
+        /** 是否有跳帧 */
+        hasFrameDropping: boolean;
+        /** 平均帧间隔（ms） */
+        averageFrameInterval: number;
+        /** 帧间隔标准差 */
+        frameIntervalStdDev: number;
+    };
+    /** 可接受的差异列表 */
+    acceptableDifferences: string[];
+    /** 不可接受的差异列表 */
+    unacceptableDifferences: string[];
+}
+export interface RetryOptions {
+    maxRetries: number;
+    retryInterval: number;
+}
+export interface DiffDetails {
+    layoutMatch: boolean;
+    colorMatch: boolean;
+    elementPositionMatch: boolean;
+    textContentMatch: boolean;
+    resourceMatch: boolean;
+    acceptableDifferences: string[];
+    unacceptableDifferences: string[];
+}
+export interface AssertSnapshot {
+    beforeScreenshot: string;
+    afterScreenshot: string;
+    timestamp: number;
+    assertion: string;
+}
 export interface AgentAssertOpt {
     keepRawResponse?: boolean;
+    /** 启用系统级校验（白屏、遮挡等），默认 true */
+    enableSystemCheck?: boolean;
+    /** 自定义系统校验规则 */
+    customSystemCheckRules?: string;
+    /** 业务知识上下文 */
+    businessContext?: string;
+    /** 手动传入操作前截图（Base64） */
+    beforeScreenshot?: string;
+    /** 手动传入操作后截图（Base64） */
+    afterScreenshot?: string;
+    /**
+     * 截图模式
+     * - 'auto': 自动判断（默认）
+     * - 'lastAction': 使用最后一次 action 的前后截图
+     * - 'flow': 使用整个流程的开始和结束截图
+     * - 'manual': 使用手动传入的截图
+     * - 'currentOnly': 仅使用当前截图（无操作前截图）
+     * - 'diff': 与基准图片进行 Diff 对比
+     * - 'video': 与基准视频进行动画对比
+     */
+    screenshotMode?: 'auto' | 'lastAction' | 'flow' | 'manual' | 'currentOnly' | 'diff' | 'video';
+    /** 基准图片列表（仅 Diff 模式） */
+    referenceImages?: ReferenceImage[];
+    /** Diff 差异阈值（0-1），默认 0.1 */
+    diffThreshold?: number;
+    /** 忽略对比的区域 */
+    ignoreRegions?: IgnoreRegion[];
+    /** 是否忽略动态内容，默认 false */
+    ignoreDynamicContent?: boolean;
+    /** 严格模式，任何差异都视为失败，默认 false */
+    strictMode?: boolean;
+    /** 视频断言配置（仅 Video 模式） */
+    videoOptions?: VideoAssertionOptions;
+    /** 当前视频输入（仅 Video 模式），用户传入的视频帧或视频 URL */
+    currentVideo?: VideoInput;
+    /** 断言重试配置 */
+    retryOptions?: RetryOptions;
+    /** 是否保存快照，默认 false */
+    saveSnapshot?: boolean;
+    /** 快照保存路径 */
+    snapshotPath?: string;
+}
+export interface ActionScreenshotContext {
+    beforeScreenshot: string;
+    afterScreenshot: string;
+    actionType: string;
+    actionParam?: any;
+    timestamp: number;
+}
+export interface SystemCheckResults {
+    whiteScreen?: boolean;
+    layoutBlocked?: boolean;
+    loadingContent?: boolean;
+    errorPrompt?: boolean;
+    backendError?: boolean;
 }
 /**
  * planning

package/package.json

@@ -1,16 +1,13 @@
 {
   "name": "@donggui/core",
-  "description": "DongGUI Core - Automate browser actions, extract data, and perform assertions using AI.",
-  "version": "1.5.14",
+  "description": "DongGUI Core - AI-powered automation SDK with enhanced assertion capabilities",
+  "version": "1.6.1",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
   "types": "./dist/types/index.d.ts",
   "module": "./dist/es/index.mjs",
-  "files": [
-    "dist",
-    "README.md"
-  ],
+  "files": ["dist", "README.md"],
   "exports": {
     ".": {
       "types": "./dist/types/index.d.ts",
@@ -60,33 +57,28 @@
   },
   "typesVersions": {
     "*": {
-      ".": [
-        "./dist/types/index.d.ts"
-      ],
-      "utils": [
-        "./dist/types/utils.d.ts"
-      ],
-      "ai-model": [
-        "./dist/types/ai-model.d.ts"
-      ],
-      "tree": [
-        "./dist/types/tree.d.ts"
-      ],
-      "device": [
-        "./dist/types/device.d.ts"
-      ],
-      "agent": [
-        "./dist/types/agent.d.ts"
-      ],
-      "yaml": [
-        "./dist/types/yaml.d.ts"
-      ],
-      "mcp": [
-        "./dist/types/skill/index.d.ts"
-      ]
+      ".": ["./dist/types/index.d.ts"],
+      "utils": ["./dist/types/utils.d.ts"],
+      "ai-model": ["./dist/types/ai-model.d.ts"],
+      "tree": ["./dist/types/tree.d.ts"],
+      "device": ["./dist/types/device.d.ts"],
+      "agent": ["./dist/types/agent.d.ts"],
+      "yaml": ["./dist/types/yaml.d.ts"],
+      "mcp": ["./dist/types/skill/index.d.ts"]
     }
   },
+  "scripts": {
+    "dev": "npm run build:watch",
+    "build": "rslib build",
+    "build:watch": "USE_DEV_REPORT=1 rslib build --watch --no-clean",
+    "test": "vitest --run",
+    "test:u": "vitest --run -u",
+    "test:ai": "AITEST=true npm run test",
+    "computer": "TEST_COMPUTER=true npm run test:ai -- tests/ai/evaluate/computer.test.ts",
+    "test:parse-action": "npm run test:ai -- tests/ai/parse-action.test.ts"
+  },
   "dependencies": {
+    "@midscene/shared": "workspace:*",
     "@ui-tars/action-parser": "1.2.3",
     "dayjs": "^1.11.11",
     "dotenv": "^16.4.5",
@@ -96,8 +88,7 @@
     "openai": "6.3.0",
     "semver": "7.5.2",
     "undici": "^6.0.0",
-    "zod": "3.24.3",
-    "@midscene/shared": "1.5.6"
+    "zod": "3.24.3"
   },
   "devDependencies": {
     "@rslib/core": "^0.18.3",
@@ -116,15 +107,5 @@
     "access": "public",
     "registry": "https://registry.npmjs.org"
   },
-  "license": "MIT",
-  "scripts": {
-    "dev": "npm run build:watch",
-    "build": "rslib build",
-    "build:watch": "USE_DEV_REPORT=1 rslib build --watch --no-clean",
-    "test": "vitest --run",
-    "test:u": "vitest --run -u",
-    "test:ai": "AITEST=true npm run test",
-    "computer": "TEST_COMPUTER=true npm run test:ai -- tests/ai/evaluate/computer.test.ts",
-    "test:parse-action": "npm run test:ai -- tests/ai/parse-action.test.ts"
-  }
-}
+  "license": "MIT"
+}