@midscene/web 0.19.1 → 0.20.0

package/dist/types/{page-90e9f9a7.d.ts → page-471361cd.d.ts}

@@ -1,10 +1,11 @@
 import { Page as Page$2 } from 'playwright';
 import * as _midscene_core from '@midscene/core';
-import { BaseElement, Rect, UIContext, PlaywrightParserOpt, PlanningLocateParam, ElementTreeNode, ExecutionDump, Size, Point, PageType } from '@midscene/core';
+import { BaseElement, Rect, Insight, ExecutionTaskProgressOptions, PlanningAction, ExecutionTaskApply, Executor, MidsceneYamlFlowItem, InsightExtractParam, InsightExtractOption, InsightAssertionResponse, PlanningActionParamWaitFor, UIContext, PlaywrightParserOpt, PlanningLocateParam, ElementTreeNode, ExecutionDump, Size, Point, PageType } from '@midscene/core';
 import { ElementInfo, ElementNode } from '@midscene/shared/extractor';
 import { Page as Page$1, KeyInput } from 'puppeteer';
 import { DebugFunction } from '@midscene/shared/logger';
 import { NodeType } from '@midscene/shared/constants';
+import { ChatCompletionMessageParam } from '@midscene/core/ai-model';
 
 declare class WebElementInfo implements BaseElement {
     content: string;
@@ -32,6 +33,103 @@ declare class WebElementInfo implements BaseElement {
     });
 }
 
+interface PlanningCache {
+    type: 'plan';
+    prompt: string;
+    yamlWorkflow: string;
+}
+interface LocateCache {
+    type: 'locate';
+    prompt: string;
+    xpaths: string[];
+}
+interface MatchCacheResult<T extends PlanningCache | LocateCache> {
+    cacheContent: T;
+    updateFn: (cb: (cache: T) => void) => void;
+}
+type CacheFileContent = {
+    midsceneVersion: string;
+    cacheId: string;
+    caches: Array<PlanningCache | LocateCache>;
+};
+declare class TaskCache {
+    cacheId: string;
+    cacheFilePath?: string;
+    cache: CacheFileContent;
+    isCacheResultUsed: boolean;
+    cacheOriginalLength: number;
+    private matchedCacheIndices;
+    constructor(cacheId: string, isCacheResultUsed: boolean, cacheFilePath?: string);
+    matchCache(prompt: string, type: 'plan' | 'locate'): MatchCacheResult<PlanningCache | LocateCache> | undefined;
+    matchPlanCache(prompt: string): MatchCacheResult<PlanningCache> | undefined;
+    matchLocateCache(prompt: string): MatchCacheResult<LocateCache> | undefined;
+    appendCache(cache: PlanningCache | LocateCache): void;
+    loadCacheFromFile(): CacheFileContent | undefined;
+    flushCacheToFile(): void;
+    updateOrAppendCacheRecord(newRecord: PlanningCache | LocateCache, cachedRecord?: MatchCacheResult<PlanningCache | LocateCache>): void;
+}
+
+interface ExecutionResult<OutputType = any> {
+    output: OutputType;
+    executor: Executor;
+}
+declare class PageTaskExecutor {
+    page: WebPage$1;
+    insight: Insight<WebElementInfo, WebUIContext>;
+    taskCache?: TaskCache;
+    conversationHistory: ChatCompletionMessageParam[];
+    onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];
+    constructor(page: WebPage$1, insight: Insight<WebElementInfo, WebUIContext>, opts: {
+        taskCache?: TaskCache;
+        onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];
+    });
+    private recordScreenshot;
+    private getElementXpath;
+    private prependExecutorWithScreenshot;
+    convertPlanToExecutable(plans: PlanningAction[], opts?: {
+        cacheable?: boolean;
+    }): Promise<{
+        tasks: ExecutionTaskApply<any, any, any, any>[];
+    }>;
+    private setupPlanningContext;
+    loadYamlFlowAsPlanning(userInstruction: string, yamlString: string): Promise<{
+        executor: Executor;
+    }>;
+    private planningTaskFromPrompt;
+    private planningTaskToGoal;
+    runPlans(title: string, plans: PlanningAction[], opts?: {
+        cacheable?: boolean;
+    }): Promise<ExecutionResult>;
+    action(userPrompt: string, actionContext?: string, opts?: {
+        cacheable?: boolean;
+    }): Promise<ExecutionResult<{
+        yamlFlow?: MidsceneYamlFlowItem[];
+    } | undefined>>;
+    actionToGoal(userPrompt: string, opts?: {
+        cacheable?: boolean;
+    }): Promise<ExecutionResult<{
+        yamlFlow?: MidsceneYamlFlowItem[];
+    } | undefined>>;
+    private createTypeQueryTask;
+    query(demand: InsightExtractParam, opt?: InsightExtractOption): Promise<ExecutionResult>;
+    boolean(prompt: string, opt?: InsightExtractOption): Promise<ExecutionResult<boolean>>;
+    number(prompt: string, opt?: InsightExtractOption): Promise<ExecutionResult<number>>;
+    string(prompt: string, opt?: InsightExtractOption): Promise<ExecutionResult<string>>;
+    assert(assertion: string): Promise<ExecutionResult<InsightAssertionResponse>>;
+    /**
+     * Append a message to the conversation history
+     * For user messages with images:
+     * - Keep max 4 user image messages in history
+     * - Remove oldest user image message when limit reached
+     * For assistant messages:
+     * - Simply append to history
+     * @param conversationHistory Message to append
+     */
+    private appendConversationHistory;
+    private appendErrorPlan;
+    waitFor(assertion: string, opt: PlanningActionParamWaitFor): Promise<ExecutionResult<void>>;
+}
+
 type WebUIContext = UIContext<WebElementInfo> & {
     url: string;
 };
@@ -48,6 +146,7 @@ declare const ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED = "NOT_IMPLEMENTED_AS_DESIG
 declare function replaceIllegalPathCharsAndSpace(str: string): string;
 declare function forceClosePopup(page: Page$1 | Page$2, debug: DebugFunction): void;
 declare function matchElementFromPlan(planLocateParam: PlanningLocateParam, tree: ElementTreeNode<BaseElement>): any;
+declare function matchElementFromCache(taskExecutor: PageTaskExecutor, xpaths: string[] | undefined, cachePrompt: string, cacheable: boolean | undefined): Promise<any>;
 declare function trimContextByViewport(execution: ExecutionDump): {
     tasks: {
         type: any;
@@ -60,7 +159,7 @@ declare function trimContextByViewport(execution: ExecutionDump): {
         output?: any;
         log?: any;
         recorder?: _midscene_core.ExecutionRecorderItem[];
-        cache?: _midscene_core.TaskCacheInfo;
+        hitBy?: _midscene_core.ExecutionTaskHitBy;
         status: "pending" | "running" | "finished" | "failed" | "cancelled";
         error?: string;
         errorStack?: string;
@@ -368,4 +467,4 @@ declare class WebPage extends Page<'playwright', Page$2> {
     constructor(page: Page$2);
 }
 
-export { type AndroidDevicePage as A, ChromeExtensionProxyPage as C, ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED as E, type PuppeteerPageOpt as P, StaticPage as S, type WebPage$1 as W, type AndroidDeviceInputOpt as a, AbstractPage as b, WebPage$2 as c, WebPage as d, WebElementInfo as e, type WebUIContext as f, type ChromePageDestroyOptions as g, printReportMsg as h, getCurrentExecutionFile as i, generateCacheId as j, replaceIllegalPathCharsAndSpace as k, forceClosePopup as l, matchElementFromPlan as m, parseContextFromWebPage as p, reportFileName as r, trimContextByViewport as t };
+export { type AndroidDevicePage as A, ChromeExtensionProxyPage as C, ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED as E, type PuppeteerPageOpt as P, StaticPage as S, TaskCache as T, type WebPage$1 as W, type AndroidDeviceInputOpt as a, AbstractPage as b, WebPage$2 as c, WebElementInfo as d, type WebUIContext as e, PageTaskExecutor as f, WebPage as g, type ChromePageDestroyOptions as h, printReportMsg as i, getCurrentExecutionFile as j, generateCacheId as k, replaceIllegalPathCharsAndSpace as l, forceClosePopup as m, matchElementFromPlan as n, matchElementFromCache as o, parseContextFromWebPage as p, reportFileName as r, trimContextByViewport as t };

package/dist/types/playground.d.ts

@@ -1,5 +1,5 @@
-import { S as StaticPage } from './page-90e9f9a7.js';
-export { E as ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED } from './page-90e9f9a7.js';
+import { S as StaticPage } from './page-471361cd.js';
+export { E as ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED } from './page-471361cd.js';
 import { PageAgent } from './agent.js';
 import 'playwright';
 import '@midscene/core';

package/dist/types/playwright.d.ts

@@ -1,14 +1,14 @@
 import { PageAgent, PageAgentOpt } from './agent.js';
 import { Page } from 'playwright';
-import { d as WebPage } from './page-90e9f9a7.js';
+import { g as WebPage } from './page-471361cd.js';
 import { AgentWaitForOpt } from '@midscene/core';
 import { TestInfo } from '@playwright/test';
 export { overrideAIConfig } from '@midscene/shared/env';
-import '@midscene/core/ai-model';
 import '@midscene/shared/extractor';
 import 'puppeteer';
 import '@midscene/shared/logger';
 import '@midscene/shared/constants';
+import '@midscene/core/ai-model';
 
 declare const PlaywrightAiFixture: (options?: {
     forceSameTabNavigation?: boolean;
@@ -62,6 +62,9 @@ declare const PlaywrightAiFixture: (options?: {
     aiBoolean: ({ page }: {
         page: Page;
     }, use: any, testInfo: TestInfo) => Promise<void>;
+    aiAsk: ({ page }: {
+        page: Page;
+    }, use: any, testInfo: TestInfo) => Promise<void>;
 };
 type PlayWrightAiFixtureType = {
     agentForPage: (page?: any, opts?: any) => Promise<PageAgent>;
@@ -80,6 +83,7 @@ type PlayWrightAiFixtureType = {
     aiNumber: (...args: Parameters<PageAgent['aiNumber']>) => ReturnType<PageAgent['aiNumber']>;
     aiString: (...args: Parameters<PageAgent['aiString']>) => ReturnType<PageAgent['aiString']>;
     aiBoolean: (...args: Parameters<PageAgent['aiBoolean']>) => ReturnType<PageAgent['aiBoolean']>;
+    aiAsk: (...args: Parameters<PageAgent['aiAsk']>) => ReturnType<PageAgent['aiAsk']>;
 };
 
 declare class PlaywrightAgent extends PageAgent<WebPage> {

package/dist/types/puppeteer-agent-launcher.d.ts

@@ -2,7 +2,7 @@ import * as puppeteer from 'puppeteer';
 import { PuppeteerAgent } from './puppeteer.js';
 import { MidsceneYamlScriptWebEnv } from '@midscene/core';
 import './agent.js';
-import './page-90e9f9a7.js';
+import './page-471361cd.js';
 import 'playwright';
 import '@midscene/shared/extractor';
 import '@midscene/shared/logger';

package/dist/types/puppeteer.d.ts

@@ -1,14 +1,14 @@
 import { PageAgentOpt, PageAgent } from './agent.js';
 import { Page } from 'puppeteer';
-import { P as PuppeteerPageOpt, c as WebPage } from './page-90e9f9a7.js';
-export { a as AndroidDeviceInputOpt } from './page-90e9f9a7.js';
+import { P as PuppeteerPageOpt, c as WebPage } from './page-471361cd.js';
+export { a as AndroidDeviceInputOpt } from './page-471361cd.js';
 export { overrideAIConfig } from '@midscene/shared/env';
 import '@midscene/core';
-import '@midscene/core/ai-model';
 import 'playwright';
 import '@midscene/shared/extractor';
 import '@midscene/shared/logger';
 import '@midscene/shared/constants';
+import '@midscene/core/ai-model';
 
 type PuppeteerAgentOpt = PageAgentOpt & PuppeteerPageOpt;
 declare class PuppeteerAgent extends PageAgent<WebPage> {

package/dist/types/utils.d.ts

@@ -2,6 +2,7 @@ import '@midscene/core';
 import '@midscene/shared/logger';
 import 'playwright';
 import 'puppeteer';
-export { E as ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED, f as WebUIContext, l as forceClosePopup, j as generateCacheId, i as getCurrentExecutionFile, m as matchElementFromPlan, p as parseContextFromWebPage, h as printReportMsg, k as replaceIllegalPathCharsAndSpace, r as reportFileName, t as trimContextByViewport } from './page-90e9f9a7.js';
+export { E as ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED, e as WebUIContext, m as forceClosePopup, k as generateCacheId, j as getCurrentExecutionFile, o as matchElementFromCache, n as matchElementFromPlan, p as parseContextFromWebPage, i as printReportMsg, l as replaceIllegalPathCharsAndSpace, r as reportFileName, t as trimContextByViewport } from './page-471361cd.js';
 import '@midscene/shared/extractor';
 import '@midscene/shared/constants';
+import '@midscene/core/ai-model';

package/dist/types/yaml.d.ts

@@ -1,7 +1,7 @@
 export { S as ScriptPlayer, p as parseYamlScript } from './utils-399ab32c.js';
 import { MidsceneYamlScriptWebEnv, MidsceneYamlTask } from '@midscene/core';
 import './agent.js';
-import './page-90e9f9a7.js';
+import './page-471361cd.js';
 import 'playwright';
 import '@midscene/shared/extractor';
 import 'puppeteer';

package/package.json

@@ -8,7 +8,7 @@
     "Browser use",
     "Android use"
   ],
-  "version": "0.19.1",
+  "version": "0.20.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "jsnext:source": "./src/index.ts",
@@ -167,8 +167,8 @@
     "semver": "7.5.2",
     "socket.io": "^4.8.1",
     "socket.io-client": "4.8.1",
-    "@midscene/core": "0.19.1",
-    "@midscene/shared": "0.19.1"
+    "@midscene/core": "0.20.0",
+    "@midscene/shared": "0.20.0"
   },
   "devDependencies": {
     "@modern-js/module-tools": "2.60.6",