@midscene/web 0.16.4 → 0.16.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent.js +78 -14
- package/dist/es/agent.js.map +1 -1
- package/dist/es/bridge-mode-browser.js +3 -3
- package/dist/es/bridge-mode-browser.js.map +1 -1
- package/dist/es/bridge-mode.js +82 -18
- package/dist/es/bridge-mode.js.map +1 -1
- package/dist/es/chrome-extension.js +79 -15
- package/dist/es/chrome-extension.js.map +1 -1
- package/dist/es/index.js +110 -14
- package/dist/es/index.js.map +1 -1
- package/dist/es/midscene-playground.js +78 -14
- package/dist/es/midscene-playground.js.map +1 -1
- package/dist/es/playground.js +78 -14
- package/dist/es/playground.js.map +1 -1
- package/dist/es/playwright.js +110 -14
- package/dist/es/playwright.js.map +1 -1
- package/dist/es/puppeteer-agent-launcher.js +78 -14
- package/dist/es/puppeteer-agent-launcher.js.map +1 -1
- package/dist/es/puppeteer.js +78 -14
- package/dist/es/puppeteer.js.map +1 -1
- package/dist/es/ui-utils.js.map +1 -1
- package/dist/lib/agent.js +77 -10
- package/dist/lib/agent.js.map +1 -1
- package/dist/lib/bridge-mode-browser.js +3 -3
- package/dist/lib/bridge-mode-browser.js.map +1 -1
- package/dist/lib/bridge-mode.js +81 -14
- package/dist/lib/bridge-mode.js.map +1 -1
- package/dist/lib/chrome-extension.js +78 -11
- package/dist/lib/chrome-extension.js.map +1 -1
- package/dist/lib/index.js +109 -10
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene-playground.js +77 -10
- package/dist/lib/midscene-playground.js.map +1 -1
- package/dist/lib/playground.js +77 -10
- package/dist/lib/playground.js.map +1 -1
- package/dist/lib/playwright.js +109 -10
- package/dist/lib/playwright.js.map +1 -1
- package/dist/lib/puppeteer-agent-launcher.js +77 -10
- package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
- package/dist/lib/puppeteer.js +77 -10
- package/dist/lib/puppeteer.js.map +1 -1
- package/dist/lib/ui-utils.js.map +1 -1
- package/dist/types/agent.d.ts +8 -0
- package/dist/types/playwright.d.ts +16 -0
- package/dist/types/ui-utils.d.ts +1 -1
- package/package.json +4 -5
package/dist/lib/ui-utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAWO,SAAS,QAAQ,MAAqB;AAC3C,SAAO,KAAK,UAAU,GAAG,KAAK,IAAI,MAAM,KAAK,WAAW,EAAE,KAAK,KAAK;AACtE;AAEO,SAAS,eACd,OAC0C;AAE1C,QAAM,OAAO,MAAM,QAAQ,KAAK,IAAI,QAAQ,CAAC,KAAK;AAIlD,SAAO,KAAK,OAAO,CAAC,KAA+C,MAAM;AACvE,UAAM,cAAc,KAAK,SAAS,MAAM,KAAK,KAAK,SAAS,SAAS;AACpE,QAAI,gBAAgB,MAAM,OAAO,MAAM,MAAM;AAC3C,aAAO,IAAI,OAAO,CAAC,EAAE,KAAK,GAAG,SAAS,YAAY,CAAC,CAAC;AAAA,IACtD;AACA,QAAI,gBAAgB,MAAM,OAAO,MAAM,MAAM;AAC3C,aAAO,IAAI,OAAO,CAAC,EAAE,KAAK,GAAG,SAAS,OAAO,CAAC,CAAC;AAAA,IACjD;AACA,QAAI,gBAAgB,MAAM,OAAO,MAAM,MAAM;AAC3C,aAAO,IAAI,OAAO,CAAC,EAAE,KAAK,GAAG,SAAS,QAAQ,CAAC,CAAC;AAAA,IAClD;AACA,WAAO,IAAI,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AAAA,EAChC,GAAG,CAAC,CAAC;AACP;AAEO,SAAS,eAAe,QAA8B;AAC3D,MAAI,CAAC,QAAQ;AACX,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,WAAW,UAAU;AAC9B,WAAO;AAAA,EACT;AAEA,SAAO,OAAO;AAChB;AAEO,SAAS,eAAe,aAAyC;AACtE,MAAI,CAAC,aAAa;AAChB,WAAO;AAAA,EACT;AACA,SAAO,GAAG,YAAY,aAAa,MAAM,KAAK,YAAY,cAAc,MAAM,KAAK,YAAY,YAAY,kBAAkB;AAC/H;AAEO,SAAS,aACd,
|
|
1
|
+
{"version":3,"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAWO,SAAS,QAAQ,MAAqB;AAC3C,SAAO,KAAK,UAAU,GAAG,KAAK,IAAI,MAAM,KAAK,WAAW,EAAE,KAAK,KAAK;AACtE;AAEO,SAAS,eACd,OAC0C;AAE1C,QAAM,OAAO,MAAM,QAAQ,KAAK,IAAI,QAAQ,CAAC,KAAK;AAIlD,SAAO,KAAK,OAAO,CAAC,KAA+C,MAAM;AACvE,UAAM,cAAc,KAAK,SAAS,MAAM,KAAK,KAAK,SAAS,SAAS;AACpE,QAAI,gBAAgB,MAAM,OAAO,MAAM,MAAM;AAC3C,aAAO,IAAI,OAAO,CAAC,EAAE,KAAK,GAAG,SAAS,YAAY,CAAC,CAAC;AAAA,IACtD;AACA,QAAI,gBAAgB,MAAM,OAAO,MAAM,MAAM;AAC3C,aAAO,IAAI,OAAO,CAAC,EAAE,KAAK,GAAG,SAAS,OAAO,CAAC,CAAC;AAAA,IACjD;AACA,QAAI,gBAAgB,MAAM,OAAO,MAAM,MAAM;AAC3C,aAAO,IAAI,OAAO,CAAC,EAAE,KAAK,GAAG,SAAS,QAAQ,CAAC,CAAC;AAAA,IAClD;AACA,WAAO,IAAI,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AAAA,EAChC,GAAG,CAAC,CAAC;AACP;AAEO,SAAS,eAAe,QAA8B;AAC3D,MAAI,CAAC,QAAQ;AACX,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,WAAW,UAAU;AAC9B,WAAO;AAAA,EACT;AAEA,SAAO,OAAO;AAChB;AAEO,SAAS,eAAe,aAAyC;AACtE,MAAI,CAAC,aAAa;AAChB,WAAO;AAAA,EACT;AACA,SAAO,GAAG,YAAY,aAAa,MAAM,KAAK,YAAY,cAAc,MAAM,KAAK,YAAY,YAAY,kBAAkB;AAC/H;AAEO,SAAS,aACd,MAcA,QACA;AACA,MAAI,QAAQ;AACV,WAAO,GAAG,IAAI,MAAM,MAAM;AAAA,EAC5B;AACA,SAAO;AACT;AAEO,SAAS,SAAS,MAAqB;AAC5C,MAAI;AACJ,MAAI,KAAK,SAAS,YAAY;AAC5B,YAAS,MAAgC,OAAO;AAAA,EAClD;AAEA,MAAI,KAAK,SAAS,WAAW;AAC3B,YACG,MAAqC,OAAO,UAC5C,MAAqC,OAAO,MAC5C,MAAoC,OAAO,cAC3C,MAAwC,OAAO;AAAA,EACpD;AAEA,MAAI,KAAK,SAAS,UAAU;AAC1B,UAAM,SAAU,MAA8B;AAC9C,UAAM,YAAY,SAAS,eAAe,MAAM,IAAI;AAEpD,YAAQ,KAAK,WAAW;AACxB,QAAI,OAAQ,MAA8B,OAAO,WAAW,UAAU;AACpE,cAAQ,GAAI,MAA8B,OAAO,MAAM;AAAA,IACzD,WACE,OAAQ,MAA8B,OAAO,eAAe,UAC5D;AACA,cAAQ,eAAgB,MAA8B,KAAK;AAAA,IAC7D,WACE,OAAQ,MAA8B,OAAO,UAAU,aACvD;AACA,cAAS,MAA8B,OAAO;AAAA,IAChD;AAEA,QAAI,WAAW;AACb,UAAI,OAAO;AACT,gBAAQ,GAAG,SAAS,MAAM,KAAK;AAAA,MACjC,OAAO;AACL,gBAAQ;AAAA,MACV;AAAA,IACF;AAAA,EACF;AAEA,MAAI,OAAO,UAAU;AAAa,WAAO;AACzC,SAAO,OAAO,UAAU,WACpB,QACA,KAAK,UAAU,OAAO,QAAW,CAAC;AACxC;AAEO,IAAM,wBAAwB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA","names":[],"ignoreList":[],"sources":["../../src/common/ui-utils.ts"],"sourcesContent":["import type {\n DetailedLocateParam,\n ExecutionTask,\n ExecutionTaskAction,\n ExecutionTaskInsightAssertion,\n ExecutionTaskInsightLocate,\n ExecutionTaskInsightQuery,\n ExecutionTaskPlanning,\n PlanningActionParamScroll,\n} from '@midscene/core';\n\nexport function typeStr(task: ExecutionTask) {\n return task.subType ? `${task.type} / ${task.subType || ''}` : task.type;\n}\n\nexport function getKeyCommands(\n value: string | string[],\n): Array<{ key: string; command?: string }> {\n // Ensure value is an array of keys\n const keys = Array.isArray(value) ? value : [value];\n\n // Process each key to attach a corresponding command if needed, based on the presence of 'Meta' or 'Control' in the keys array.\n // ref: https://github.com/puppeteer/puppeteer/pull/9357/files#diff-32cf475237b000f980eb214a0a823e45a902bddb7d2426d677cae96397aa0ae4R94\n return keys.reduce((acc: Array<{ key: string; command?: string }>, k) => {\n const includeMeta = keys.includes('Meta') || keys.includes('Control');\n if (includeMeta && (k === 'a' || k === 'A')) {\n return acc.concat([{ key: k, command: 'SelectAll' }]);\n }\n if (includeMeta && (k === 'c' || k === 'C')) {\n return acc.concat([{ key: k, command: 'Copy' }]);\n }\n if (includeMeta && (k === 'v' || k === 'V')) {\n return acc.concat([{ key: k, command: 'Paste' }]);\n }\n return acc.concat([{ key: k }]);\n }, []);\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam) {\n if (!locate) {\n return '';\n }\n\n if (typeof locate === 'string') {\n return locate;\n }\n\n return locate.prompt;\n}\n\nexport function scrollParamStr(scrollParam?: PlanningActionParamScroll) {\n if (!scrollParam) {\n return '';\n }\n return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'once'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function taskTitleStr(\n type:\n | 'Tap'\n | 'Hover'\n | 'Input'\n | 'KeyboardPress'\n | 'Scroll'\n | 'Action'\n | 'Query'\n | 'Assert'\n | 'WaitFor'\n | 'Locate'\n | 'Boolean'\n | 'Number'\n | 'String',\n prompt: string,\n) {\n if (prompt) {\n return `${type} - ${prompt}`;\n }\n return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n let value: string | undefined | object;\n if (task.type === 'Planning') {\n value = (task as ExecutionTaskPlanning)?.param?.userInstruction;\n }\n\n if (task.type === 'Insight') {\n value =\n (task as ExecutionTaskInsightLocate)?.param?.prompt ||\n (task as ExecutionTaskInsightLocate)?.param?.id ||\n (task as ExecutionTaskInsightQuery)?.param?.dataDemand ||\n (task as ExecutionTaskInsightAssertion)?.param?.assertion;\n }\n\n if (task.type === 'Action') {\n const locate = (task as ExecutionTaskAction)?.locate;\n const locateStr = locate ? locateParamStr(locate) : '';\n\n value = task.thought || '';\n if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n } else if (\n typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n ) {\n value = scrollParamStr((task as ExecutionTaskAction)?.param);\n } else if (\n typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n ) {\n value = (task as ExecutionTaskAction)?.param?.value;\n }\n\n if (locateStr) {\n if (value) {\n value = `${locateStr} - ${value}`;\n } else {\n value = locateStr;\n }\n }\n }\n\n if (typeof value === 'undefined') return '';\n return typeof value === 'string'\n ? value\n : JSON.stringify(value, undefined, 2);\n}\n\nexport const limitOpenNewTabScript = `\nif (!window.__MIDSCENE_NEW_TAB_INTERCEPTOR_INITIALIZED__) {\n window.__MIDSCENE_NEW_TAB_INTERCEPTOR_INITIALIZED__ = true;\n\n // Intercept the window.open method (only once)\n window.open = function(url) {\n console.log('Blocked window.open:', url);\n window.location.href = url;\n return null;\n };\n\n // Block all a tag clicks with target=\"_blank\" (only once)\n document.addEventListener('click', function(e) {\n const target = e.target.closest('a');\n if (target && target.target === '_blank') {\n e.preventDefault();\n console.log('Blocked new tab:', target.href);\n window.location.href = target.href;\n target.removeAttribute('target');\n }\n }, true);\n}\n`;\n"]}
|
package/dist/types/agent.d.ts
CHANGED
|
@@ -123,7 +123,11 @@ declare class PageTaskExecutor {
|
|
|
123
123
|
output: any;
|
|
124
124
|
executor: Executor;
|
|
125
125
|
}>;
|
|
126
|
+
private createTypeQueryTask;
|
|
126
127
|
query(demand: InsightExtractParam): Promise<ExecutionResult>;
|
|
128
|
+
boolean(prompt: string): Promise<ExecutionResult<boolean>>;
|
|
129
|
+
number(prompt: string): Promise<ExecutionResult<number>>;
|
|
130
|
+
string(prompt: string): Promise<ExecutionResult<string>>;
|
|
127
131
|
assert(assertion: string): Promise<ExecutionResult<InsightAssertionResponse>>;
|
|
128
132
|
/**
|
|
129
133
|
* Append a message to the conversation history
|
|
@@ -184,6 +188,10 @@ declare class PageAgent<PageType extends WebPage = WebPage> {
|
|
|
184
188
|
aiScroll(scrollParam: PlanningActionParamScroll, locatePrompt?: string, opt?: LocateOption): Promise<any>;
|
|
185
189
|
aiAction(taskPrompt: string): Promise<any>;
|
|
186
190
|
aiQuery(demand: any): Promise<any>;
|
|
191
|
+
aiBoolean(prompt: string): Promise<boolean>;
|
|
192
|
+
aiNumber(prompt: string): Promise<number>;
|
|
193
|
+
aiString(prompt: string): Promise<string>;
|
|
194
|
+
aiLocate(prompt: string, opt?: LocateOption): Promise<Pick<LocateResultElement, "rect" | "center">>;
|
|
187
195
|
aiAssert(assertion: string, msg?: string, opt?: AgentAssertOpt): Promise<_midscene_core.InsightAssertionResponse | undefined>;
|
|
188
196
|
aiWaitFor(assertion: string, opt?: AgentWaitForOpt): Promise<void>;
|
|
189
197
|
ai(taskPrompt: string, type?: string): Promise<any>;
|
|
@@ -48,6 +48,18 @@ declare const PlaywrightAiFixture: (options?: {
|
|
|
48
48
|
aiWaitFor: ({ page }: {
|
|
49
49
|
page: Page;
|
|
50
50
|
}, use: any, testInfo: TestInfo) => Promise<void>;
|
|
51
|
+
aiLocate: ({ page }: {
|
|
52
|
+
page: Page;
|
|
53
|
+
}, use: any, testInfo: TestInfo) => Promise<void>;
|
|
54
|
+
aiNumber: ({ page }: {
|
|
55
|
+
page: Page;
|
|
56
|
+
}, use: any, testInfo: TestInfo) => Promise<void>;
|
|
57
|
+
aiString: ({ page }: {
|
|
58
|
+
page: Page;
|
|
59
|
+
}, use: any, testInfo: TestInfo) => Promise<void>;
|
|
60
|
+
aiBoolean: ({ page }: {
|
|
61
|
+
page: Page;
|
|
62
|
+
}, use: any, testInfo: TestInfo) => Promise<void>;
|
|
51
63
|
};
|
|
52
64
|
type PlayWrightAiFixtureType = {
|
|
53
65
|
agentForPage: (page?: any, opts?: any) => Promise<PageAgent>;
|
|
@@ -61,6 +73,10 @@ type PlayWrightAiFixtureType = {
|
|
|
61
73
|
aiQuery: <T = any>(...args: Parameters<PageAgent['aiQuery']>) => ReturnType<PageAgent['aiQuery']>;
|
|
62
74
|
aiAssert: (...args: Parameters<PageAgent['aiAssert']>) => ReturnType<PageAgent['aiAssert']>;
|
|
63
75
|
aiWaitFor: (assertion: string, opt?: AgentWaitForOpt) => Promise<void>;
|
|
76
|
+
aiLocate: (...args: Parameters<PageAgent['aiLocate']>) => ReturnType<PageAgent['aiLocate']>;
|
|
77
|
+
aiNumber: (...args: Parameters<PageAgent['aiNumber']>) => ReturnType<PageAgent['aiNumber']>;
|
|
78
|
+
aiString: (...args: Parameters<PageAgent['aiString']>) => ReturnType<PageAgent['aiString']>;
|
|
79
|
+
aiBoolean: (...args: Parameters<PageAgent['aiBoolean']>) => ReturnType<PageAgent['aiBoolean']>;
|
|
64
80
|
};
|
|
65
81
|
|
|
66
82
|
declare class PlaywrightAgent extends PageAgent<WebPage> {
|
package/dist/types/ui-utils.d.ts
CHANGED
|
@@ -7,7 +7,7 @@ declare function getKeyCommands(value: string | string[]): Array<{
|
|
|
7
7
|
}>;
|
|
8
8
|
declare function locateParamStr(locate?: DetailedLocateParam): string;
|
|
9
9
|
declare function scrollParamStr(scrollParam?: PlanningActionParamScroll): string;
|
|
10
|
-
declare function taskTitleStr(type: 'Tap' | 'Hover' | 'Input' | 'KeyboardPress' | 'Scroll' | 'Action' | 'Query' | 'Assert' | 'WaitFor', prompt: string): string;
|
|
10
|
+
declare function taskTitleStr(type: 'Tap' | 'Hover' | 'Input' | 'KeyboardPress' | 'Scroll' | 'Action' | 'Query' | 'Assert' | 'WaitFor' | 'Locate' | 'Boolean' | 'Number' | 'String', prompt: string): string;
|
|
11
11
|
declare function paramStr(task: ExecutionTask): string;
|
|
12
12
|
declare const limitOpenNewTabScript = "\nif (!window.__MIDSCENE_NEW_TAB_INTERCEPTOR_INITIALIZED__) {\n window.__MIDSCENE_NEW_TAB_INTERCEPTOR_INITIALIZED__ = true;\n\n // Intercept the window.open method (only once)\n window.open = function(url) {\n console.log('Blocked window.open:', url);\n window.location.href = url;\n return null;\n };\n\n // Block all a tag clicks with target=\"_blank\" (only once)\n document.addEventListener('click', function(e) {\n const target = e.target.closest('a');\n if (target && target.target === '_blank') {\n e.preventDefault();\n console.log('Blocked new tab:', target.href);\n window.location.href = target.href;\n target.removeAttribute('target');\n }\n }, true);\n}\n";
|
|
13
13
|
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@midscene/web",
|
|
3
3
|
"description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
|
|
4
|
-
"version": "0.16.
|
|
4
|
+
"version": "0.16.6",
|
|
5
5
|
"repository": "https://github.com/web-infra-dev/midscene",
|
|
6
6
|
"homepage": "https://midscenejs.com/",
|
|
7
7
|
"jsnext:source": "./src/index.ts",
|
|
@@ -149,12 +149,12 @@
|
|
|
149
149
|
"inquirer": "10.1.5",
|
|
150
150
|
"js-sha256": "0.11.0",
|
|
151
151
|
"js-yaml": "4.1.0",
|
|
152
|
-
"
|
|
152
|
+
"fkill": "7.2.1",
|
|
153
153
|
"openai": "4.81.0",
|
|
154
154
|
"socket.io": "^4.8.1",
|
|
155
155
|
"socket.io-client": "4.8.1",
|
|
156
|
-
"@midscene/
|
|
157
|
-
"@midscene/
|
|
156
|
+
"@midscene/shared": "0.16.6",
|
|
157
|
+
"@midscene/core": "0.16.6"
|
|
158
158
|
},
|
|
159
159
|
"devDependencies": {
|
|
160
160
|
"@modern-js/module-tools": "2.60.6",
|
|
@@ -164,7 +164,6 @@
|
|
|
164
164
|
"@types/express": "^4.17.21",
|
|
165
165
|
"@types/fs-extra": "11.0.4",
|
|
166
166
|
"@types/js-yaml": "4.0.9",
|
|
167
|
-
"@types/kill-port": "2.0.3",
|
|
168
167
|
"@types/node": "^18.0.0",
|
|
169
168
|
"playwright": "1.44.1",
|
|
170
169
|
"puppeteer": "24.2.0",
|