@midscene/web 0.7.1 → 0.7.2-beta-20241024094141.0

package/dist/browser/types/playground.d.ts

@@ -0,0 +1,313 @@
+import * as _midscene_core from '@midscene/core';
+import { PlanningAction, AIElementParseResponse, Insight, InsightExtractParam, InsightAssertionResponse, PlanningActionParamWaitFor, Executor, GroupedActionDump, ExecutionDump, AgentAssertOpt, AgentWaitForOpt, BaseElement, Rect, UIContext } from '@midscene/core';
+import { KeyInput, Page as Page$2 } from 'puppeteer';
+import { NodeType } from '@midscene/shared/constants';
+import { Page as Page$3 } from 'playwright';
+import { Browser } from 'webdriverio';
+import { getRunningPkgInfo } from '@midscene/shared/fs';
+
+type PlanTask = {
+    type: 'plan';
+    prompt: string;
+    pageContext: {
+        url: string;
+        size: {
+            width: number;
+            height: number;
+        };
+    };
+    response: {
+        plans: PlanningAction[];
+    };
+};
+type LocateTask = {
+    type: 'locate';
+    prompt: string;
+    pageContext: {
+        url: string;
+        size: {
+            width: number;
+            height: number;
+        };
+    };
+    response: AIElementParseResponse;
+};
+type AiTasks = Array<PlanTask | LocateTask>;
+type AiTaskCache = {
+    aiTasks: Array<{
+        prompt: string;
+        tasks: AiTasks;
+    }>;
+};
+declare class TaskCache {
+    cache: AiTaskCache;
+    cacheId: string;
+    newCache: AiTaskCache;
+    midscenePkgInfo: ReturnType<typeof getRunningPkgInfo> | null;
+    constructor(opts?: {
+        fileName?: string;
+    });
+    getCacheGroupByPrompt(aiActionPrompt: string): {
+        readCache: <T extends "plan" | "locate">(pageContext: WebUIContext, type: T, actionPrompt: string) => false | (T extends "plan" ? {
+            plans: PlanningAction<any>[];
+        } : AIElementParseResponse);
+        saveCache: (cache: PlanTask | LocateTask) => void;
+    };
+    /**
+     * Read and return cached responses asynchronously based on specific criteria
+     * This function is mainly used to read cached responses from a certain storage medium.
+     * It accepts three parameters: the page context information, the task type, and the user's prompt information.
+     * In the function, it first checks whether there is cached data. If there is, it retrieves the first task response from the cache.
+     * It then checks whether the task type is 'locate' and whether the corresponding element can be found in the new context.
+     * If the element cannot be found, it returns false, indicating that the cache is invalid.
+     * If the task type is correct and the user prompt matches, it checks whether the page context is the same.
+     * If the page context is the same, it returns the cached response, indicating that the cache hit is successful.
+     * If there is no cached data or the conditions are not met, the function returns false, indicating that no cache is available or the cache is not hit.
+     *
+     * @param pageContext UIContext<WebElementInfo> type, representing the context information of the current page
+     * @param type String type, specifying the task type, can be 'plan' or 'locate'
+     * @param userPrompt String type, representing user prompt information
+     * @return Returns a Promise object that resolves to a boolean or object
+     */
+    readCache(pageContext: WebUIContext, type: 'plan', userPrompt: string, cacheGroup: AiTasks): PlanTask['response'];
+    readCache(pageContext: WebUIContext, type: 'locate', userPrompt: string, cacheGroup: AiTasks): LocateTask['response'];
+    pageContextEqual(taskPageContext: LocateTask['pageContext'], pageContext: WebUIContext): boolean;
+    /**
+     * Generate task cache data.
+     * This method is mainly used to create or obtain some cached data for tasks, and it returns a new cache object.
+     * In the cache object, it may contain task-related information, states, or other necessary data.
+     * It is assumed that the `newCache` property already exists in the current class or object and is a data structure used to store task cache.
+     * @returns {Object} Returns a new cache object, which may include task cache data.
+     */
+    generateTaskCache(): AiTaskCache;
+    readCacheFromFile(): AiTaskCache | undefined;
+    writeCacheToFile(): void;
+}
+
+interface ExecutionResult<OutputType = any> {
+    output: OutputType;
+    executor: Executor;
+}
+declare class PageTaskExecutor {
+    page: WebPage;
+    insight: Insight<WebElementInfo, WebUIContext>;
+    taskCache: TaskCache;
+    constructor(page: WebPage, opts: {
+        cacheId: string | undefined;
+    });
+    private recordScreenshot;
+    private wrapExecutorWithScreenshot;
+    private convertPlanToExecutable;
+    action(userPrompt: string): Promise<ExecutionResult>;
+    query(demand: InsightExtractParam): Promise<ExecutionResult>;
+    assert(assertion: string): Promise<ExecutionResult<InsightAssertionResponse>>;
+    waitFor(assertion: string, opt: PlanningActionParamWaitFor): Promise<ExecutionResult<void>>;
+}
+
+interface PageAgentOpt {
+    testId?: string;
+    cacheId?: string;
+    groupName?: string;
+    groupDescription?: string;
+    cache?: AiTaskCache;
+    generateReport?: boolean;
+    autoPrintReportMsg?: boolean;
+}
+declare class PageAgent {
+    page: WebPage;
+    dump: GroupedActionDump;
+    reportFile?: string | null;
+    reportFileName?: string;
+    taskExecutor: PageTaskExecutor;
+    opts: PageAgentOpt;
+    constructor(page: WebPage, opts?: PageAgentOpt);
+    appendExecutionDump(execution: ExecutionDump): void;
+    dumpDataString(): string;
+    writeOutActionDumps(): void;
+    aiAction(taskPrompt: string): Promise<void>;
+    aiQuery(demand: any): Promise<any>;
+    aiAssert(assertion: string, msg?: string, opt?: AgentAssertOpt): Promise<_midscene_core.AIAssertionResponse | undefined>;
+    aiWaitFor(assertion: string, opt?: AgentWaitForOpt): Promise<void>;
+    ai(taskPrompt: string, type?: string): Promise<any>;
+}
+
+interface ElementInfo {
+    id: string;
+    indexId: number;
+    nodePath: string;
+    nodeHashId: string;
+    locator: string;
+    attributes: {
+        nodeType: NodeType;
+        [key: string]: string;
+    };
+    nodeType: NodeType;
+    content: string;
+    rect: {
+        left: number;
+        top: number;
+        width: number;
+        height: number;
+    };
+    center: [number, number];
+}
+
+type MouseButton = 'left' | 'right' | 'middle';
+declare abstract class AbstractPage {
+    abstract pageType: string;
+    abstract screenshotBase64?(): Promise<string>;
+    abstract getElementInfos(): Promise<ElementInfo[]>;
+    abstract url(): string;
+    get mouse(): {
+        click: (x: number, y: number, options: {
+            button: MouseButton;
+        }) => Promise<void>;
+        wheel: (deltaX: number, deltaY: number) => Promise<void>;
+        move: (x: number, y: number) => Promise<void>;
+    };
+    get keyboard(): {
+        type: (text: string) => Promise<void>;
+        press: (key: WebKeyInput) => Promise<void>;
+    };
+    clearInput(element: ElementInfo): Promise<void>;
+    abstract scrollUntilTop(): Promise<void>;
+    abstract scrollUntilBottom(): Promise<void>;
+    abstract scrollUpOneScreen(): Promise<void>;
+    abstract scrollDownOneScreen(): Promise<void>;
+    abstract _forceUsePageContext?(): Promise<WebUIContext>;
+}
+
+type WebKeyInput$1 = KeyInput;
+declare class Page$1 implements AbstractPage {
+    private browser;
+    pageType: string;
+    constructor(browser: Browser);
+    getElementInfos(): Promise<ElementInfo[]>;
+    screenshotBase64(): Promise<string>;
+    get mouse(): {
+        click: (x: number, y: number, options?: {
+            button: MouseButton;
+        }) => Promise<void>;
+        wheel: (deltaX: number, deltaY: number) => Promise<void>;
+        move: (x: number, y: number) => Promise<void>;
+    };
+    get keyboard(): {
+        type: (text: string) => Promise<void>;
+        press: (key: WebKeyInput$1) => Promise<void>;
+    };
+    clearInput(element: ElementInfo): Promise<void>;
+    url(): string;
+    scrollUntilTop(): Promise<void>;
+    scrollUntilBottom(): Promise<void>;
+    scrollUpOneScreen(): Promise<void>;
+    scrollDownOneScreen(): Promise<void>;
+    private keyboardType;
+    private keyboardPress;
+    private mouseClick;
+    private mouseMove;
+    private mouseWheel;
+}
+
+declare class Page<AgentType extends 'puppeteer' | 'playwright', PageType extends Page$2 | Page$3> implements AbstractPage {
+    private underlyingPage;
+    pageType: AgentType;
+    private evaluate;
+    constructor(underlyingPage: PageType, pageType: AgentType);
+    getElementInfos(): Promise<ElementInfo[]>;
+    screenshotBase64(): Promise<string>;
+    url(): string;
+    get mouse(): {
+        click: (x: number, y: number, options?: {
+            button: MouseButton;
+        }) => Promise<void>;
+        wheel: (deltaX: number, deltaY: number) => Promise<void>;
+        move: (x: number, y: number) => Promise<void>;
+    };
+    get keyboard(): {
+        type: (text: string) => Promise<void>;
+        press: (key: WebKeyInput) => Promise<void>;
+        down: (key: WebKeyInput) => Promise<void>;
+        up: (key: WebKeyInput) => Promise<void>;
+    };
+    clearInput(element: ElementInfo): Promise<void>;
+    scrollUntilTop(): Promise<void>;
+    scrollUntilBottom(): Promise<void>;
+    scrollUpOneScreen(): Promise<void>;
+    scrollDownOneScreen(): Promise<void>;
+}
+
+declare class WebPage$2 extends Page<'playwright', Page$3> {
+    constructor(page: Page$3);
+}
+
+declare class WebPage$1 extends Page<'puppeteer', Page$2> {
+    constructor(page: Page$2);
+}
+
+type WebPage =
+  | WebPage$2
+  | WebPage$1
+  | Page$1
+  | StaticPage;
+type WebKeyInput = KeyInput;
+
+declare class WebElementInfo implements BaseElement {
+    content: string;
+    locator: string;
+    rect: Rect;
+    center: [number, number];
+    page: WebPage;
+    id: string;
+    indexId: number;
+    attributes: {
+        nodeType: NodeType;
+        [key: string]: string;
+    };
+    constructor({ content, rect, page, locator, id, attributes, indexId, }: {
+        content: string;
+        rect: Rect;
+        page: WebPage;
+        locator: string;
+        id: string;
+        attributes: {
+            nodeType: NodeType;
+            [key: string]: string;
+        };
+        indexId: number;
+    });
+}
+
+type WebUIContext = UIContext<WebElementInfo> & {
+    url: string;
+};
+declare const ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED = "NOT_IMPLEMENTED_AS_DESIGNED";
+
+declare class StaticPage implements AbstractPage {
+    pageType: string;
+    private uiContext;
+    constructor(uiContext: WebUIContext);
+    getElementInfos(): Promise<any>;
+    screenshotBase64(): Promise<string>;
+    url(): string;
+    scrollUntilTop(): Promise<any>;
+    scrollUntilBottom(): Promise<any>;
+    scrollUpOneScreen(): Promise<any>;
+    scrollDownOneScreen(): Promise<any>;
+    clearInput(): Promise<any>;
+    mouse: {
+        click: any;
+        wheel: any;
+        move: any;
+    };
+    keyboard: {
+        type: any;
+        press: any;
+    };
+    _forceUsePageContext(): Promise<WebUIContext>;
+}
+
+declare class StaticPageAgent extends PageAgent {
+    constructor(page: StaticPage);
+}
+
+export { ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED, StaticPage, StaticPageAgent };