@midscene/web 0.7.2 → 0.7.3-beta-20241104100519.0

package/dist/types/appium.d.ts

@@ -1,9 +1,9 @@
-export { P as AppiumAgent } from './tasks-82c1054b.js';
-export { P as AppiumPage } from './page-ad820b3c.js';
-import '@midscene/core/dist/types/types-ce502a2a';
+export { P as AppiumAgent } from './tasks-bf894fe9.js';
+export { P as AppiumPage } from './page-84b18c7f.js';
 import '@midscene/core';
 import '@midscene/shared/fs';
 import 'playwright';
 import 'puppeteer';
 import '@midscene/shared/constants';
+import '@midscene/core/.';
 import 'webdriverio';

package/dist/types/chrome-extension.d.ts

@@ -0,0 +1,16 @@
+import { C as ChromeExtensionProxyPage } from './page-84b18c7f.js';
+export { b as ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED } from './page-84b18c7f.js';
+import { P as PageAgent } from './tasks-bf894fe9.js';
+import 'playwright';
+import 'puppeteer';
+import '@midscene/shared/constants';
+import '@midscene/core';
+import '@midscene/core/.';
+import 'webdriverio';
+import '@midscene/shared/fs';
+
+declare class ChromeExtensionProxyPageAgent extends PageAgent {
+    constructor(page: ChromeExtensionProxyPage);
+}
+
+export { ChromeExtensionProxyPage, ChromeExtensionProxyPageAgent };

package/dist/types/debug.d.ts

@@ -1,9 +1,10 @@
 import { writeFileSync } from 'node:fs';
-import { W as WebPage, E as ElementInfo } from './page-ad820b3c.js';
+import { W as WebPage, E as ElementInfo } from './page-84b18c7f.js';
 import 'playwright';
 import 'puppeteer';
-import '@midscene/core';
 import '@midscene/shared/constants';
+import '@midscene/core';
+import '@midscene/core/.';
 import 'webdriverio';
 
 declare function generateExtractData(page: WebPage, targetDir: string, saveImgType?: {

package/dist/types/index.d.ts

@@ -1,12 +1,12 @@
 export { PlayWrightAiFixtureType, PlaywrightAiFixture } from './playwright.js';
-export { P as AppiumAgent, P as PlaywrightAgent } from './tasks-82c1054b.js';
-export { P as AppiumPage } from './page-ad820b3c.js';
+export { P as AppiumAgent, P as PlaywrightAgent } from './tasks-bf894fe9.js';
+export { P as AppiumPage } from './page-84b18c7f.js';
+export { StaticPageAgent } from './playground.js';
 export { PuppeteerAgent } from './puppeteer.js';
 export { generateExtractData } from './debug.js';
 import '@midscene/core/.';
 import '@playwright/test';
 import 'playwright';
-import '@midscene/core/dist/types/types-ce502a2a';
 import '@midscene/core';
 import '@midscene/shared/fs';
 import 'puppeteer';

package/dist/types/{page-ad820b3c.d.ts → page-84b18c7f.d.ts}

@@ -1,12 +1,13 @@
 import { Page as Page$3 } from 'playwright';
 import { KeyInput, Page as Page$2 } from 'puppeteer';
-import { BaseElement, Rect, UIContext } from '@midscene/core';
 import { NodeType } from '@midscene/shared/constants';
+import { BaseElement, Rect, UIContext, PlaywrightParserOpt } from '@midscene/core';
+import * as _midscene_core__ from '@midscene/core/.';
 import { Browser } from 'webdriverio';
 
 declare class WebElementInfo implements BaseElement {
     content: string;
-    locator: string;
+    locator?: string;
     rect: Rect;
     center: [number, number];
     page: WebPage$1;
@@ -20,7 +21,7 @@ declare class WebElementInfo implements BaseElement {
         content: string;
         rect: Rect;
         page: WebPage$1;
-        locator: string;
+        locator?: string;
         id: string;
         attributes: {
             nodeType: NodeType;
@@ -33,6 +34,17 @@ declare class WebElementInfo implements BaseElement {
 type WebUIContext = UIContext<WebElementInfo> & {
     url: string;
 };
+declare function parseContextFromWebPage(page: WebPage$1, _opt?: PlaywrightParserOpt): Promise<WebUIContext>;
+declare function getExtraReturnLogic(): Promise<string>;
+declare function reportFileName(tag?: string): string;
+declare function printReportMsg(filepath: string): void;
+/**
+ * Get the current execution file name
+ * @returns The name of the current execution file
+ */
+declare function getCurrentExecutionFile(trace?: string): string | false;
+declare function generateCacheId(fileName?: string): string;
+declare const ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED = "NOT_IMPLEMENTED_AS_DESIGNED";
 
 interface ElementInfo {
     id: string;
@@ -58,9 +70,14 @@ interface ElementInfo {
 type MouseButton = 'left' | 'right' | 'middle';
 declare abstract class AbstractPage {
     abstract pageType: string;
-    abstract screenshot(): Promise<string>;
     abstract getElementInfos(): Promise<ElementInfo[]>;
-    abstract url(): string;
+    abstract url(): string | Promise<string>;
+    abstract screenshotBase64?(): Promise<string>;
+    abstract size(): Promise<{
+        width: number;
+        height: number;
+        dpr?: number;
+    }>;
     get mouse(): {
         click: (x: number, y: number, options: {
             button: MouseButton;
@@ -78,6 +95,7 @@ declare abstract class AbstractPage {
     abstract scrollUpOneScreen(): Promise<void>;
     abstract scrollDownOneScreen(): Promise<void>;
     abstract _forceUsePageContext?(): Promise<WebUIContext>;
+    abstract destroy(): Promise<void>;
 }
 
 type WebKeyInput$1 = KeyInput;
@@ -86,7 +104,11 @@ declare class Page$1 implements AbstractPage {
     pageType: string;
     constructor(browser: Browser);
     getElementInfos(): Promise<ElementInfo[]>;
-    screenshot(): Promise<string>;
+    size(): Promise<{
+        width: number;
+        height: number;
+    }>;
+    screenshotBase64(): Promise<string>;
     get mouse(): {
         click: (x: number, y: number, options?: {
             button: MouseButton;
@@ -109,6 +131,42 @@ declare class Page$1 implements AbstractPage {
     private mouseClick;
     private mouseMove;
     private mouseWheel;
+    destroy(): Promise<void>;
+}
+
+declare class ChromeExtensionProxyPage implements AbstractPage {
+    pageType: string;
+    private tabId;
+    private windowId;
+    private viewportSize?;
+    private debuggerAttached;
+    constructor(tabId: number, windowId: number);
+    private attachDebugger;
+    private detachDebugger;
+    private sendCommandToDebugger;
+    getElementInfos(): Promise<ElementInfo[]>;
+    size(): Promise<{
+        width: number;
+        height: number;
+        dpr: number;
+    }>;
+    screenshotBase64(): Promise<string>;
+    url(): Promise<string>;
+    scrollUntilTop(): Promise<void>;
+    scrollUntilBottom(): Promise<void>;
+    scrollUpOneScreen(): Promise<void>;
+    scrollDownOneScreen(): Promise<void>;
+    clearInput(element: ElementInfo): Promise<void>;
+    mouse: {
+        click: (x: number, y: number) => Promise<void>;
+        wheel: (deltaX: number, deltaY: number) => Promise<void>;
+        move: (x: number, y: number) => Promise<void>;
+    };
+    keyboard: {
+        type: (text: string) => Promise<void>;
+        press: (key: WebKeyInput) => Promise<void>;
+    };
+    destroy(): Promise<void>;
 }
 
 declare class StaticPage implements AbstractPage {
@@ -116,8 +174,9 @@ declare class StaticPage implements AbstractPage {
     private uiContext;
     constructor(uiContext: WebUIContext);
     getElementInfos(): Promise<any>;
-    screenshot(): Promise<string>;
-    url(): string;
+    size(): Promise<_midscene_core__.Size>;
+    screenshotBase64(): Promise<string>;
+    url(): Promise<string>;
     scrollUntilTop(): Promise<any>;
     scrollUntilBottom(): Promise<any>;
     scrollUpOneScreen(): Promise<any>;
@@ -133,16 +192,23 @@ declare class StaticPage implements AbstractPage {
         press: any;
     };
     _forceUsePageContext(): Promise<WebUIContext>;
+    destroy(): Promise<void>;
 }
 
 declare class Page<AgentType extends 'puppeteer' | 'playwright', PageType extends Page$2 | Page$3> implements AbstractPage {
-    private page;
+    private underlyingPage;
+    private viewportSize?;
     pageType: AgentType;
     private evaluate;
-    constructor(page: PageType, pageType: AgentType);
+    constructor(underlyingPage: PageType, pageType: AgentType);
     getElementInfos(): Promise<ElementInfo[]>;
-    screenshot(): Promise<string>;
-    url(): string;
+    size(): Promise<{
+        width: number;
+        height: number;
+        dpr: number;
+    }>;
+    screenshotBase64(): Promise<string>;
+    url(): Promise<string>;
     get mouse(): {
         click: (x: number, y: number, options?: {
             button: MouseButton;
@@ -161,6 +227,7 @@ declare class Page<AgentType extends 'puppeteer' | 'playwright', PageType extend
     scrollUntilBottom(): Promise<void>;
     scrollUpOneScreen(): Promise<void>;
     scrollDownOneScreen(): Promise<void>;
+    destroy(): Promise<void>;
 }
 
 declare class WebPage$2 extends Page<'puppeteer', Page$2> {
@@ -171,11 +238,12 @@ type WebPage$1 =
   | WebPage
   | WebPage$2
   | Page$1
-  | StaticPage;
+  | StaticPage
+  | ChromeExtensionProxyPage;
 type WebKeyInput = KeyInput;
 
 declare class WebPage extends Page<'playwright', Page$3> {
     constructor(page: Page$3);
 }
 
-export { type ElementInfo as E, Page$1 as P, StaticPage as S, type WebPage$1 as W, WebPage$2 as a, WebPage as b, type WebUIContext as c, WebElementInfo as d };
+export { ChromeExtensionProxyPage as C, type ElementInfo as E, Page$1 as P, StaticPage as S, type WebPage$1 as W, WebPage$2 as a, ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED as b, WebPage as c, type WebUIContext as d, WebElementInfo as e, printReportMsg as f, getExtraReturnLogic as g, getCurrentExecutionFile as h, generateCacheId as i, parseContextFromWebPage as p, reportFileName as r };

package/dist/types/playground.d.ts

@@ -1,30 +1,16 @@
-import { P as PageAgent } from './tasks-82c1054b.js';
-import { S as StaticPage } from './page-ad820b3c.js';
-import * as http from 'http';
-import { Server } from 'node:http';
-import express from 'express';
-import '@midscene/core/dist/types/types-ce502a2a';
-import '@midscene/core';
-import '@midscene/shared/fs';
+import { S as StaticPage } from './page-84b18c7f.js';
+export { b as ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED } from './page-84b18c7f.js';
+import { P as PageAgent } from './tasks-bf894fe9.js';
 import 'playwright';
 import 'puppeteer';
 import '@midscene/shared/constants';
+import '@midscene/core';
+import '@midscene/core/.';
 import 'webdriverio';
+import '@midscene/shared/fs';
 
 declare class StaticPageAgent extends PageAgent {
     constructor(page: StaticPage);
 }
 
-declare class PlaygroundServer {
-    app: express.Application;
-    tmpDir: string;
-    server?: Server;
-    port?: number | null;
-    constructor();
-    filePathForUuid(uuid: string): string;
-    saveContextFile(uuid: string, context: string): string;
-    launch(): Promise<unknown>;
-    close(): Server<typeof http.IncomingMessage, typeof http.ServerResponse> | undefined;
-}
-
-export { PlaygroundServer, StaticPage, StaticPageAgent };
+export { StaticPage, StaticPageAgent };

package/dist/types/playwright.d.ts

@@ -1,10 +1,9 @@
 import { AgentWaitForOpt } from '@midscene/core/.';
 import { TestInfo } from '@playwright/test';
 import { Page } from 'playwright';
-import { b as PageTaskExecutor } from './tasks-82c1054b.js';
-export { P as PlaywrightAgent } from './tasks-82c1054b.js';
-export { b as PlaywrightWebPage } from './page-ad820b3c.js';
-import '@midscene/core/dist/types/types-ce502a2a';
+import { b as PageTaskExecutor } from './tasks-bf894fe9.js';
+export { P as PlaywrightAgent } from './tasks-bf894fe9.js';
+export { c as PlaywrightWebPage } from './page-84b18c7f.js';
 import '@midscene/core';
 import '@midscene/shared/fs';
 import 'puppeteer';

package/dist/types/puppeteer.d.ts

@@ -1,11 +1,11 @@
-import { P as PageAgent, a as PageAgentOpt } from './tasks-82c1054b.js';
+import { P as PageAgent, a as PageAgentOpt } from './tasks-bf894fe9.js';
 import { Page } from 'puppeteer';
-export { a as PuppeteerWebPage } from './page-ad820b3c.js';
-import '@midscene/core/dist/types/types-ce502a2a';
+export { a as PuppeteerWebPage } from './page-84b18c7f.js';
 import '@midscene/core';
 import '@midscene/shared/fs';
 import 'playwright';
 import '@midscene/shared/constants';
+import '@midscene/core/.';
 import 'webdriverio';
 
 declare class PuppeteerAgent extends PageAgent {

package/dist/types/{tasks-82c1054b.d.ts → tasks-bf894fe9.d.ts}

@@ -1,7 +1,7 @@
-import * as _midscene_core_dist_types_types_ce502a2a from '@midscene/core/dist/types/types-ce502a2a';
-import { c as WebUIContext, W as WebPage, d as WebElementInfo } from './page-ad820b3c.js';
-import { PlanningAction, AIElementParseResponse, GroupedActionDump, ExecutionDump, AgentAssertOpt, AgentWaitForOpt, Insight, InsightExtractParam, InsightAssertionResponse, PlanningActionParamWaitFor, Executor } from '@midscene/core';
-import { getMidscenePkgInfo } from '@midscene/shared/fs';
+import * as _midscene_core from '@midscene/core';
+import { PlanningAction, AIElementIdResponse, Insight, GroupedActionDump, ExecutionDump, AgentAssertOpt, AgentWaitForOpt, InsightExtractParam, InsightAssertionResponse, PlanningActionParamWaitFor, Executor } from '@midscene/core';
+import { d as WebUIContext, W as WebPage, e as WebElementInfo } from './page-84b18c7f.js';
+import { getRunningPkgInfo } from '@midscene/shared/fs';
 
 type PlanTask = {
     type: 'plan';
@@ -27,7 +27,7 @@ type LocateTask = {
             height: number;
         };
     };
-    response: AIElementParseResponse;
+    response: AIElementIdResponse;
 };
 type AiTasks = Array<PlanTask | LocateTask>;
 type AiTaskCache = {
@@ -40,14 +40,14 @@ declare class TaskCache {
     cache: AiTaskCache;
     cacheId: string;
     newCache: AiTaskCache;
-    midscenePkgInfo: ReturnType<typeof getMidscenePkgInfo>;
+    midscenePkgInfo: ReturnType<typeof getRunningPkgInfo> | null;
     constructor(opts?: {
         fileName?: string;
     });
     getCacheGroupByPrompt(aiActionPrompt: string): {
         readCache: <T extends "plan" | "locate">(pageContext: WebUIContext, type: T, actionPrompt: string) => false | (T extends "plan" ? {
             plans: PlanningAction<any>[];
-        } : AIElementParseResponse);
+        } : AIElementIdResponse);
         saveCache: (cache: PlanTask | LocateTask) => void;
     };
     /**
@@ -92,20 +92,29 @@ interface PageAgentOpt {
 }
 declare class PageAgent {
     page: WebPage;
+    insight: Insight<WebElementInfo, WebUIContext>;
     dump: GroupedActionDump;
-    reportFile?: string;
+    reportFile?: string | null;
     reportFileName?: string;
     taskExecutor: PageTaskExecutor;
     opts: PageAgentOpt;
+    /**
+     * If true, the agent will not perform any actions
+     */
+    dryMode: boolean;
     constructor(page: WebPage, opts?: PageAgentOpt);
+    getUIContext(): Promise<WebUIContext>;
+    resetDump(): GroupedActionDump;
     appendExecutionDump(execution: ExecutionDump): void;
     dumpDataString(): string;
+    reportHTMLString(): string;
     writeOutActionDumps(): void;
     aiAction(taskPrompt: string): Promise<void>;
     aiQuery(demand: any): Promise<any>;
-    aiAssert(assertion: string, msg?: string, opt?: AgentAssertOpt): Promise<_midscene_core_dist_types_types_ce502a2a.d | undefined>;
+    aiAssert(assertion: string, msg?: string, opt?: AgentAssertOpt): Promise<_midscene_core.AIAssertionResponse | undefined>;
     aiWaitFor(assertion: string, opt?: AgentWaitForOpt): Promise<void>;
     ai(taskPrompt: string, type?: string): Promise<any>;
+    destroy(): Promise<void>;
 }
 
 interface ExecutionResult<OutputType = any> {
@@ -116,7 +125,7 @@ declare class PageTaskExecutor {
     page: WebPage;
     insight: Insight<WebElementInfo, WebUIContext>;
     taskCache: TaskCache;
-    constructor(page: WebPage, opts: {
+    constructor(page: WebPage, insight: Insight<WebElementInfo, WebUIContext>, opts: {
         cacheId: string | undefined;
     });
     private recordScreenshot;

package/dist/types/utils.d.ts

@@ -0,0 +1,7 @@
+import '@midscene/core';
+export { b as ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED, d as WebUIContext, i as generateCacheId, h as getCurrentExecutionFile, g as getExtraReturnLogic, p as parseContextFromWebPage, f as printReportMsg, r as reportFileName } from './page-84b18c7f.js';
+import 'playwright';
+import 'puppeteer';
+import '@midscene/shared/constants';
+import '@midscene/core/.';
+import 'webdriverio';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/web",
   "description": "An AI-powered automation SDK can control the page, perform assertions, and extract data in JSON format using natural language. See https://midscenejs.com/ for details.",
-  "version": "0.7.2",
+  "version": "0.7.3-beta-20241104100519.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "jsnext:source": "./src/index.ts",
@@ -12,19 +12,24 @@
   },
   "exports": {
     ".": "./dist/lib/index.js",
+    "./utils": "./dist/lib/utils.js",
     "./puppeteer": "./dist/lib/puppeteer.js",
     "./playwright": "./dist/lib/playwright.js",
     "./playwright-report": "./dist/lib/playwright-report.js",
     "./playground": "./dist/lib/playground.js",
     "./debug": "./dist/lib/debug.js",
     "./constants": "./dist/lib/constants.js",
-    "./html-element": "./dist/lib/html-element/index.js"
+    "./html-element": "./dist/lib/html-element/index.js",
+    "./chrome-extension": "./dist/lib/chrome-extension.js"
   },
   "typesVersions": {
     "*": {
       ".": [
         "./dist/types/index.d.ts"
       ],
+      "utils": [
+        "./dist/types/utils.d.ts"
+      ],
       "puppeteer": [
         "./dist/types/puppeteer.d.ts"
       ],
@@ -45,6 +50,9 @@
       ],
       "html-element": [
         "./dist/types/html-element/index.d.ts"
+      ],
+      "chrome-extension": [
+        "./dist/types/chrome-extension.d.ts"
       ]
     }
   },
@@ -60,12 +68,13 @@
     "inquirer": "10.1.5",
     "cors": "2.8.5",
     "@xmldom/xmldom": "0.8.10",
-    "@midscene/shared": "0.7.2",
-    "@midscene/core": "0.7.2"
+    "@midscene/shared": "0.7.3-beta-20241104100519.0",
+    "@midscene/core": "0.7.3-beta-20241104100519.0"
   },
   "devDependencies": {
-    "@modern-js/module-tools": "2.58.2",
+    "@modern-js/module-tools": "2.60.6",
     "@playwright/test": "1.44.1",
+    "@types/chrome": "0.0.279",
     "@types/cors": "2.8.12",
     "@types/fs-extra": "11.0.4",
     "@types/node": "^18.0.0",