@rpascene/shared 0.30.9 → 0.30.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -3
- package/dist/es/common.mjs +7 -7
- package/dist/es/constants/example-code.mjs +10 -10
- package/dist/es/env/init-debug.mjs +2 -2
- package/dist/es/env/model-config-manager.mjs +1 -1
- package/dist/es/extractor/debug.mjs +2 -2
- package/dist/es/extractor/util.mjs +11 -11
- package/dist/es/extractor/web-extractor.mjs +7 -7
- package/dist/es/logger.mjs +3 -3
- package/dist/es/node/fs.mjs +6 -6
- package/dist/es/utils.mjs +2 -2
- package/dist/lib/common.js +15 -15
- package/dist/lib/constants/example-code.js +10 -10
- package/dist/lib/env/init-debug.js +2 -2
- package/dist/lib/env/model-config-manager.js +1 -1
- package/dist/lib/extractor/debug.js +1 -1
- package/dist/lib/extractor/util.js +16 -16
- package/dist/lib/extractor/web-extractor.js +6 -6
- package/dist/lib/logger.js +2 -2
- package/dist/lib/node/fs.js +6 -6
- package/dist/lib/utils.js +2 -2
- package/dist/types/common.d.ts +5 -5
- package/dist/types/constants/example-code.d.ts +2 -2
- package/dist/types/env/types.d.ts +1 -1
- package/dist/types/extractor/util.d.ts +2 -2
- package/dist/types/node/fs.d.ts +1 -1
- package/package.json +12 -12
- package/src/common.ts +7 -7
- package/src/constants/example-code.ts +10 -10
- package/src/env/init-debug.ts +2 -2
- package/src/env/model-config-manager.ts +1 -1
- package/src/env/types.ts +1 -1
- package/src/extractor/debug.ts +2 -2
- package/src/extractor/util.ts +15 -15
- package/src/extractor/web-extractor.ts +7 -7
- package/src/logger.ts +3 -3
- package/src/node/fs.ts +5 -5
- package/src/utils.ts +2 -2
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export declare const PLAYWRIGHT_EXAMPLE_CODE = "\n// Reference the following code to generate
|
|
2
|
-
export declare const YAML_EXAMPLE_CODE = "\n1. Format:\n\nweb:\n url: \"starting_url\"\n viewportWidth: 1280\n viewportHeight: 960\n\ntasks:\n - name: \"descriptive task name\"\n flow:\n - aiTap: \"element description\"\n - aiInput: 'text value'\n locate: 'input field description'\n - aiScroll:\n direction: down/up\n scrollType: untilBottom/untilTop/page\n - aiAssert: \"expected state\"\n - sleep: milliseconds\n\n2. Action Types:\n- aiTap: for clicks (natural language targeting)\n- aiInput: for text input with 'locate' field\n- aiScroll: with direction and scrollType\n- aiAssert: for validations\n- sleep: for delays (milliseconds)\n\n3. Best Practices:\n- Group related actions into logical tasks\n- Use natural language descriptions\n- Add deepThink: true for complex interactions\n- Keep task names concise but descriptive\n\n\n\nYAML type\ntasks:\n - name: <name>\n continueOnError: <boolean> # Optional, whether to continue to the next task on error, defaults to false.\n flow:\n # Auto Planning (.ai)\n # ----------------\n\n # Perform an interaction. `ai` is a shorthand for `aiAction`.\n - ai: <prompt>\n cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.\n\n # This usage is the same as `ai`.\n - aiAction: <prompt>\n cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.\n\n # Instant Action (.aiTap, .aiDoubleClick, .aiHover, .aiInput, .aiKeyboardPress, .aiScroll)\n # ----------------\n\n # Tap an element described by a prompt.\n - aiTap: <prompt>\n deepThink: <boolean> # Optional, whether to use deepThink to precisely locate the element. Defaults to False.\n xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided,
|
|
1
|
+
export declare const PLAYWRIGHT_EXAMPLE_CODE = "\n// Reference the following code to generate Rpascene test cases\n// The following is test code for Rpascene AI, for reference\n// The following is Playwright syntax, you can use Playwright to assist in test generation\nIMPORTANT: Follow these exact type signatures for AI functions:\n\n// Type signatures for AI functions:\naiInput(value: string, locator: string): Promise<void>\naiTap(locator: string): Promise<void>\naiDoubleClick(locator: string): Promise<void>\naiScroll(scrollParam: {\n direction: 'up' | 'down' | 'left' | 'right',\n scrollType: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft',\n distance: number - scroll distance, px is the unit\n}): Promise<void>\naiAssert(assertion: string): Promise<void>\naiQuery<T>(queryObject: Record<string, string>): Promise<T> // Extracts data from page based on descriptions\n\n// examples:\n// Reference the following code to generate Rpascene test cases\n// The following is test code for Rpascene AI, for reference\n// The following is Playwright syntax, you can use Playwright to assist in test generation\nimport { test as base } from '@playwright/test';\nimport type { PlayWrightAiFixtureType } from '@rpascene/web/playwright';\nimport { PlaywrightAiFixture } from '@rpascene/web/playwright';\n\nconst test = base.extend<PlayWrightAiFixtureType>(PlaywrightAiFixture({\n waitForNetworkIdleTimeout: 2000, // optional, the timeout for waiting for network idle between each action, default is 2000ms\n}));\n\n\ntest.beforeEach(async ({ page }) => {\n await page.goto('https://www.xxx.com/');\n await page.setViewportSize({ width: 1920, height: 1080 });\n});\n\ntest('ai shop', async ({\n aiInput,\n aiAssert,\n aiQuery,\n aiKeyboardPress,\n aiHover,\n aiTap,\n agentForPage,\n page,\n}) => {\n // login\n await aiAssert('The page shows the login interface');\n await aiInput('user_name', 'in user name input');\n await aiInput('password', 'in password input');\n await aiKeyboardPress('Enter', 'Login Button');\n\n // check the login success\n await aiWaitFor('The page shows that the loading is complete');\n await aiAssert('The current page shows the product detail page');\n\n // check the product info\n const dataA = await aiQuery({\n userInfo: 'User information in the format {name: string}',\n theFirstProductInfo: 'The first product info in the format {name: string, price: number}',\n });\n expect(dataA.theFirstProductInfo.name).toBe('xxx');\n expect(dataA.theFirstProductInfo.price).toBe(100);\n\n\n // add to cart\n await aiTap('click add to cart button');\n \n await aiTap('click right top cart icon');\n await aiAssert('The cart icon shows the number 1');\n});\n";
|
|
2
|
+
export declare const YAML_EXAMPLE_CODE = "\n1. Format:\n\nweb:\n url: \"starting_url\"\n viewportWidth: 1280\n viewportHeight: 960\n\ntasks:\n - name: \"descriptive task name\"\n flow:\n - aiTap: \"element description\"\n - aiInput: 'text value'\n locate: 'input field description'\n - aiScroll:\n direction: down/up\n scrollType: untilBottom/untilTop/page\n - aiAssert: \"expected state\"\n - sleep: milliseconds\n\n2. Action Types:\n- aiTap: for clicks (natural language targeting)\n- aiInput: for text input with 'locate' field\n- aiScroll: with direction and scrollType\n- aiAssert: for validations\n- sleep: for delays (milliseconds)\n\n3. Best Practices:\n- Group related actions into logical tasks\n- Use natural language descriptions\n- Add deepThink: true for complex interactions\n- Keep task names concise but descriptive\n\n\n\nYAML type\ntasks:\n - name: <name>\n continueOnError: <boolean> # Optional, whether to continue to the next task on error, defaults to false.\n flow:\n # Auto Planning (.ai)\n # ----------------\n\n # Perform an interaction. `ai` is a shorthand for `aiAction`.\n - ai: <prompt>\n cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.\n\n # This usage is the same as `ai`.\n - aiAction: <prompt>\n cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.\n\n # Instant Action (.aiTap, .aiDoubleClick, .aiHover, .aiInput, .aiKeyboardPress, .aiScroll)\n # ----------------\n\n # Tap an element described by a prompt.\n - aiTap: <prompt>\n deepThink: <boolean> # Optional, whether to use deepThink to precisely locate the element. Defaults to False.\n xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Rpascene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.\n cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.\n\n # Double click an element described by a prompt.\n - aiDoubleClick: <prompt>\n deepThink: <boolean> # Optional, whether to use deepThink to precisely locate the element. Defaults to False.\n xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Rpascene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.\n cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.\n\n # Hover over an element described by a prompt.\n - aiHover: <prompt>\n deepThink: <boolean> # Optional, whether to use deepThink to precisely locate the element. Defaults to False.\n xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Rpascene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.\n cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.\n\n # Input text into an element described by a prompt.\n - aiInput: <final text content of the input>\n locate: <prompt>\n deepThink: <boolean> # Optional, whether to use deepThink to precisely locate the element. Defaults to False.\n xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Rpascene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.\n cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.\n\n # Press a key (e.g., Enter, Tab, Escape) on an element described by a prompt.\n - aiKeyboardPress: <key>\n locate: <prompt>\n deepThink: <boolean> # Optional, whether to use deepThink to precisely locate the element. Defaults to False.\n xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Rpascene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.\n cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.\n\n # Scroll globally or on an element described by a prompt.\n - aiScroll:\n direction: 'up' # or 'down' | 'left' | 'right'\n scrollType: 'once' # or 'untilTop' | 'untilBottom' | 'untilLeft' | 'untilRight'\n distance: <number> # Optional, the scroll distance in pixels.\n locate: <prompt> # Optional, the element to scroll on.\n deepThink: <boolean> # Optional, whether to use deepThink to precisely locate the element. Defaults to False.\n xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Rpascene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.\n cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.\n\n # Log the current screenshot with a description in the report file.\n - logScreenshot: <title> # Optional, the title of the screenshot. If not provided, the title will be 'untitled'.\n content: <content> # Optional, the description of the screenshot.\n\n # Data Extraction\n # ----------------\n\n # Perform a query that returns a JSON object.\n - aiQuery: <prompt> # Remember to describe the format of the result in the prompt.\n name: <name> # The key for the query result in the JSON output.\n\n # More APIs\n # ----------------\n\n # Wait for a condition to be met, with a timeout (in ms, optional, defaults to 30000).\n - aiWaitFor: <prompt>\n timeout: <ms>\n\n # Perform an assertion.\n - aiAssert: <prompt>\n errorMessage: <error-message> # Optional, the error message to print if the assertion fails.\n\n # Wait for a specified amount of time.\n - sleep: <ms>\n\n # Execute a piece of JavaScript code in the web page context.\n - javascript: <javascript>\n name: <name> # Optional, assign a name to the return value, which will be used as a key in the JSON output.\n\n - name: <name>\n flow:\n # ...\n";
|
|
@@ -118,7 +118,7 @@ export declare const BOOLEAN_ENV_KEYS: readonly ["MIDSCENE_CACHE", "MIDSCENE_LAN
|
|
|
118
118
|
export declare const NUMBER_ENV_KEYS: readonly ["MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT"];
|
|
119
119
|
export declare const STRING_ENV_KEYS: readonly ["OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER"];
|
|
120
120
|
/**
|
|
121
|
-
* Non model related env keys, used for globally controlling the behavior of
|
|
121
|
+
* Non model related env keys, used for globally controlling the behavior of rpascene
|
|
122
122
|
* Can not be override by agent.modelConfig but can be override by overrideAIConfig
|
|
123
123
|
* Can be access at any time
|
|
124
124
|
*/
|
|
@@ -32,12 +32,12 @@ export declare function elementRect(el: globalThis.HTMLElement | globalThis.Node
|
|
|
32
32
|
} | false;
|
|
33
33
|
export declare function validTextNodeContent(node: globalThis.Node): string | false;
|
|
34
34
|
export declare function getNodeAttributes(node: globalThis.HTMLElement | globalThis.Node, currentWindow: typeof globalThis.window): Record<string, string>;
|
|
35
|
-
export declare function
|
|
35
|
+
export declare function rpasceneGenerateHash(node: globalThis.Node | null, content: string, rect: Rect): string;
|
|
36
36
|
export declare function setNodeHashCacheListOnWindow(): void;
|
|
37
37
|
export declare function setNodeToCacheList(node: globalThis.Node, id: string): void;
|
|
38
38
|
export declare function getNodeFromCacheList(id: string): any;
|
|
39
39
|
export declare function generateId(numberId: number): string;
|
|
40
40
|
export declare function setGenerateHashOnWindow(): void;
|
|
41
|
-
export declare function
|
|
41
|
+
export declare function setRpasceneVisibleRectOnWindow(): void;
|
|
42
42
|
export declare function setExtractTextWithPositionOnWindow(): void;
|
|
43
43
|
export declare function getTopDocument(): globalThis.HTMLElement;
|
package/dist/types/node/fs.d.ts
CHANGED
|
@@ -11,5 +11,5 @@ export declare function getRunningPkgInfo(dir?: string): PkgInfo | null;
|
|
|
11
11
|
*/
|
|
12
12
|
export declare function findNearestPackageJson(dir: string): string | null;
|
|
13
13
|
export declare function getElementInfosScriptContent(): string;
|
|
14
|
-
export declare function getExtraReturnLogic(tree?: boolean): Promise<";
|
|
14
|
+
export declare function getExtraReturnLogic(tree?: boolean): Promise<";rpascene_element_inspector.setNodeHashCacheListOnWindow();;rpascene_element_inspector.webExtractNodeTree()" | ";rpascene_element_inspector.setNodeHashCacheListOnWindow();;rpascene_element_inspector.webExtractTextWithPosition()" | null>;
|
|
15
15
|
export {};
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rpascene/shared",
|
|
3
3
|
"description": "RPA shared",
|
|
4
|
-
"version": "0.30.
|
|
4
|
+
"version": "0.30.11",
|
|
5
5
|
"repository": "",
|
|
6
6
|
"homepage": "",
|
|
7
7
|
"types": "./dist/types/index.d.ts",
|
|
@@ -69,16 +69,6 @@
|
|
|
69
69
|
"src",
|
|
70
70
|
"README.md"
|
|
71
71
|
],
|
|
72
|
-
"scripts": {
|
|
73
|
-
"dev": "npm run build:watch",
|
|
74
|
-
"build": "npm run build:script && npm run build:pkg",
|
|
75
|
-
"build:pkg": "rslib build",
|
|
76
|
-
"build:script": "rslib build -c ./rslib.inspect.config.ts",
|
|
77
|
-
"build:watch": "npm run build:script && rslib build --watch",
|
|
78
|
-
"reset": "rimraf ./**/node_modules",
|
|
79
|
-
"test": "vitest --run",
|
|
80
|
-
"test:u": "vitest --run -u"
|
|
81
|
-
},
|
|
82
72
|
"dependencies": {
|
|
83
73
|
"@silvia-odwyer/photon": "0.3.3",
|
|
84
74
|
"@silvia-odwyer/photon-node": "0.3.3",
|
|
@@ -103,5 +93,15 @@
|
|
|
103
93
|
"access": "public",
|
|
104
94
|
"registry": "https://registry.npmjs.org/"
|
|
105
95
|
},
|
|
106
|
-
"license": "MIT"
|
|
96
|
+
"license": "MIT",
|
|
97
|
+
"scripts": {
|
|
98
|
+
"dev": "npm run build:watch",
|
|
99
|
+
"build": "npm run build:script && npm run build:pkg",
|
|
100
|
+
"build:pkg": "rslib build",
|
|
101
|
+
"build:script": "rslib build -c ./rslib.inspect.config.ts",
|
|
102
|
+
"build:watch": "npm run build:script && rslib build --watch",
|
|
103
|
+
"reset": "rimraf ./**/node_modules",
|
|
104
|
+
"test": "vitest --run",
|
|
105
|
+
"test:u": "vitest --run -u"
|
|
106
|
+
}
|
|
107
107
|
}
|
package/src/common.ts
CHANGED
|
@@ -7,10 +7,10 @@ import { getBasicEnvValue } from './env/basic';
|
|
|
7
7
|
import { MIDSCENE_RUN_DIR } from './env/types';
|
|
8
8
|
import { ifInNode } from './utils';
|
|
9
9
|
|
|
10
|
-
export const defaultRunDirName = '
|
|
10
|
+
export const defaultRunDirName = 'rpascene_run';
|
|
11
11
|
// Define locally for now to avoid import issues
|
|
12
12
|
|
|
13
|
-
export const
|
|
13
|
+
export const getRpasceneRunDir = () => {
|
|
14
14
|
if (!ifInNode) {
|
|
15
15
|
return '';
|
|
16
16
|
}
|
|
@@ -18,12 +18,12 @@ export const getMidsceneRunDir = () => {
|
|
|
18
18
|
return getBasicEnvValue(MIDSCENE_RUN_DIR) || defaultRunDirName;
|
|
19
19
|
};
|
|
20
20
|
|
|
21
|
-
export const
|
|
21
|
+
export const getRpasceneRunBaseDir = () => {
|
|
22
22
|
if (!ifInNode) {
|
|
23
23
|
return '';
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
-
let basePath = path.resolve(process.cwd(),
|
|
26
|
+
let basePath = path.resolve(process.cwd(), getRpasceneRunDir());
|
|
27
27
|
|
|
28
28
|
// Create a base directory
|
|
29
29
|
if (!existsSync(basePath)) {
|
|
@@ -40,13 +40,13 @@ export const getMidsceneRunBaseDir = () => {
|
|
|
40
40
|
};
|
|
41
41
|
|
|
42
42
|
/**
|
|
43
|
-
* Get the path to the
|
|
43
|
+
* Get the path to the rpascene_run directory or a subdirectory within it.
|
|
44
44
|
* Creates the directory if it doesn't exist.
|
|
45
45
|
*
|
|
46
46
|
* @param subdir - Optional subdirectory name (e.g., 'log', 'report')
|
|
47
47
|
* @returns The absolute path to the requested directory
|
|
48
48
|
*/
|
|
49
|
-
export const
|
|
49
|
+
export const getRpasceneRunSubDir = (
|
|
50
50
|
subdir: 'dump' | 'cache' | 'report' | 'tmp' | 'log' | 'output',
|
|
51
51
|
): string => {
|
|
52
52
|
if (!ifInNode) {
|
|
@@ -54,7 +54,7 @@ export const getMidsceneRunSubDir = (
|
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
// Create a log directory
|
|
57
|
-
const basePath =
|
|
57
|
+
const basePath = getRpasceneRunBaseDir();
|
|
58
58
|
const logPath = path.join(basePath, subdir);
|
|
59
59
|
if (!existsSync(logPath)) {
|
|
60
60
|
mkdirSync(logPath, { recursive: true });
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export const PLAYWRIGHT_EXAMPLE_CODE = `
|
|
2
|
-
// Reference the following code to generate
|
|
3
|
-
// The following is test code for
|
|
2
|
+
// Reference the following code to generate Rpascene test cases
|
|
3
|
+
// The following is test code for Rpascene AI, for reference
|
|
4
4
|
// The following is Playwright syntax, you can use Playwright to assist in test generation
|
|
5
5
|
IMPORTANT: Follow these exact type signatures for AI functions:
|
|
6
6
|
|
|
@@ -17,8 +17,8 @@ aiAssert(assertion: string): Promise<void>
|
|
|
17
17
|
aiQuery<T>(queryObject: Record<string, string>): Promise<T> // Extracts data from page based on descriptions
|
|
18
18
|
|
|
19
19
|
// examples:
|
|
20
|
-
// Reference the following code to generate
|
|
21
|
-
// The following is test code for
|
|
20
|
+
// Reference the following code to generate Rpascene test cases
|
|
21
|
+
// The following is test code for Rpascene AI, for reference
|
|
22
22
|
// The following is Playwright syntax, you can use Playwright to assist in test generation
|
|
23
23
|
import { test as base } from '@playwright/test';
|
|
24
24
|
import type { PlayWrightAiFixtureType } from '@rpascene/web/playwright';
|
|
@@ -128,33 +128,33 @@ tasks:
|
|
|
128
128
|
# Tap an element described by a prompt.
|
|
129
129
|
- aiTap: <prompt>
|
|
130
130
|
deepThink: <boolean> # Optional, whether to use deepThink to precisely locate the element. Defaults to False.
|
|
131
|
-
xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided,
|
|
131
|
+
xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Rpascene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.
|
|
132
132
|
cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.
|
|
133
133
|
|
|
134
134
|
# Double click an element described by a prompt.
|
|
135
135
|
- aiDoubleClick: <prompt>
|
|
136
136
|
deepThink: <boolean> # Optional, whether to use deepThink to precisely locate the element. Defaults to False.
|
|
137
|
-
xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided,
|
|
137
|
+
xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Rpascene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.
|
|
138
138
|
cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.
|
|
139
139
|
|
|
140
140
|
# Hover over an element described by a prompt.
|
|
141
141
|
- aiHover: <prompt>
|
|
142
142
|
deepThink: <boolean> # Optional, whether to use deepThink to precisely locate the element. Defaults to False.
|
|
143
|
-
xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided,
|
|
143
|
+
xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Rpascene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.
|
|
144
144
|
cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.
|
|
145
145
|
|
|
146
146
|
# Input text into an element described by a prompt.
|
|
147
147
|
- aiInput: <final text content of the input>
|
|
148
148
|
locate: <prompt>
|
|
149
149
|
deepThink: <boolean> # Optional, whether to use deepThink to precisely locate the element. Defaults to False.
|
|
150
|
-
xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided,
|
|
150
|
+
xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Rpascene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.
|
|
151
151
|
cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.
|
|
152
152
|
|
|
153
153
|
# Press a key (e.g., Enter, Tab, Escape) on an element described by a prompt.
|
|
154
154
|
- aiKeyboardPress: <key>
|
|
155
155
|
locate: <prompt>
|
|
156
156
|
deepThink: <boolean> # Optional, whether to use deepThink to precisely locate the element. Defaults to False.
|
|
157
|
-
xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided,
|
|
157
|
+
xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Rpascene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.
|
|
158
158
|
cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.
|
|
159
159
|
|
|
160
160
|
# Scroll globally or on an element described by a prompt.
|
|
@@ -164,7 +164,7 @@ tasks:
|
|
|
164
164
|
distance: <number> # Optional, the scroll distance in pixels.
|
|
165
165
|
locate: <prompt> # Optional, the element to scroll on.
|
|
166
166
|
deepThink: <boolean> # Optional, whether to use deepThink to precisely locate the element. Defaults to False.
|
|
167
|
-
xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided,
|
|
167
|
+
xpath: <xpath> # Optional, the xpath of the target element for the operation. If provided, Rpascene will prioritize this xpath to find the element before using the cache and the AI model. Defaults to empty.
|
|
168
168
|
cacheable: <boolean> # Optional, whether to cache the result of this API call when the [caching feature](./caching.mdx) is enabled. Defaults to True.
|
|
169
169
|
|
|
170
170
|
# Log the current screenshot with a description in the report file.
|
package/src/env/init-debug.ts
CHANGED
|
@@ -7,7 +7,7 @@ export const initDebugConfig = () => {
|
|
|
7
7
|
let debugConfig = '';
|
|
8
8
|
if (shouldPrintTiming) {
|
|
9
9
|
console.warn(
|
|
10
|
-
'MIDSCENE_DEBUG_AI_PROFILE is deprecated, use DEBUG=
|
|
10
|
+
'MIDSCENE_DEBUG_AI_PROFILE is deprecated, use DEBUG=rpascene:ai:profile instead',
|
|
11
11
|
);
|
|
12
12
|
debugConfig = 'ai:profile';
|
|
13
13
|
}
|
|
@@ -15,7 +15,7 @@ export const initDebugConfig = () => {
|
|
|
15
15
|
|
|
16
16
|
if (shouldPrintAIResponse) {
|
|
17
17
|
console.warn(
|
|
18
|
-
'MIDSCENE_DEBUG_AI_RESPONSE is deprecated, use DEBUG=
|
|
18
|
+
'MIDSCENE_DEBUG_AI_RESPONSE is deprecated, use DEBUG=rpascene:ai:response instead',
|
|
19
19
|
);
|
|
20
20
|
if (debugConfig) {
|
|
21
21
|
debugConfig = 'ai:*';
|
|
@@ -138,7 +138,7 @@ export class ModelConfigManager {
|
|
|
138
138
|
|
|
139
139
|
if (!modelConfig.vlMode) {
|
|
140
140
|
throw new Error(
|
|
141
|
-
'No visual language model (VL model) detected for the current scenario. Element localization may be inaccurate. Please verify your model configuration. Learn more: https://
|
|
141
|
+
'No visual language model (VL model) detected for the current scenario. Element localization may be inaccurate. Please verify your model configuration. Learn more: https://rpascenejs.com/choose-a-model',
|
|
142
142
|
);
|
|
143
143
|
}
|
|
144
144
|
}
|
package/src/env/types.ts
CHANGED
|
@@ -218,7 +218,7 @@ export const STRING_ENV_KEYS = [
|
|
|
218
218
|
] as const;
|
|
219
219
|
|
|
220
220
|
/**
|
|
221
|
-
* Non model related env keys, used for globally controlling the behavior of
|
|
221
|
+
* Non model related env keys, used for globally controlling the behavior of rpascene
|
|
222
222
|
* Can not be override by agent.modelConfig but can be override by overrideAIConfig
|
|
223
223
|
* Can be access at any time
|
|
224
224
|
*/
|
package/src/extractor/debug.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { webExtractTextWithPosition } from '.';
|
|
2
2
|
import {
|
|
3
3
|
setExtractTextWithPositionOnWindow,
|
|
4
|
-
|
|
4
|
+
setRpasceneVisibleRectOnWindow,
|
|
5
5
|
} from './util';
|
|
6
6
|
|
|
7
7
|
console.log(webExtractTextWithPosition(document.body, true));
|
|
8
8
|
console.log(JSON.stringify(webExtractTextWithPosition(document.body, true)));
|
|
9
9
|
setExtractTextWithPositionOnWindow();
|
|
10
|
-
|
|
10
|
+
setRpasceneVisibleRectOnWindow();
|
package/src/extractor/util.ts
CHANGED
|
@@ -225,13 +225,13 @@ export function elementRect(
|
|
|
225
225
|
baseZoom = 1,
|
|
226
226
|
):
|
|
227
227
|
| {
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
228
|
+
left: number;
|
|
229
|
+
top: number;
|
|
230
|
+
width: number;
|
|
231
|
+
height: number;
|
|
232
|
+
zoom: number;
|
|
233
|
+
isVisible: boolean;
|
|
234
|
+
}
|
|
235
235
|
| false {
|
|
236
236
|
if (!el) {
|
|
237
237
|
logger(el, 'Element is not in the DOM hierarchy');
|
|
@@ -392,7 +392,7 @@ export function getNodeAttributes(
|
|
|
392
392
|
return Object.fromEntries(attributesList);
|
|
393
393
|
}
|
|
394
394
|
|
|
395
|
-
export function
|
|
395
|
+
export function rpasceneGenerateHash(
|
|
396
396
|
node: globalThis.Node | null,
|
|
397
397
|
content: string,
|
|
398
398
|
rect: Rect,
|
|
@@ -400,7 +400,7 @@ export function midsceneGenerateHash(
|
|
|
400
400
|
const slicedHash = generateHashId(rect, content);
|
|
401
401
|
|
|
402
402
|
if (node) {
|
|
403
|
-
if (!(window as any).
|
|
403
|
+
if (!(window as any).rpasceneNodeHashCacheList) {
|
|
404
404
|
setNodeHashCacheListOnWindow();
|
|
405
405
|
}
|
|
406
406
|
|
|
@@ -413,7 +413,7 @@ export function midsceneGenerateHash(
|
|
|
413
413
|
|
|
414
414
|
export function setNodeHashCacheListOnWindow() {
|
|
415
415
|
if (typeof window !== 'undefined') {
|
|
416
|
-
(window as any).
|
|
416
|
+
(window as any).rpasceneNodeHashCacheList = [];
|
|
417
417
|
}
|
|
418
418
|
}
|
|
419
419
|
|
|
@@ -422,13 +422,13 @@ export function setNodeToCacheList(node: globalThis.Node, id: string) {
|
|
|
422
422
|
if (getNodeFromCacheList(id)) {
|
|
423
423
|
return;
|
|
424
424
|
}
|
|
425
|
-
(window as any).
|
|
425
|
+
(window as any).rpasceneNodeHashCacheList?.push({ node, id });
|
|
426
426
|
}
|
|
427
427
|
}
|
|
428
428
|
|
|
429
429
|
export function getNodeFromCacheList(id: string) {
|
|
430
430
|
if (typeof window !== 'undefined') {
|
|
431
|
-
return (window as any).
|
|
431
|
+
return (window as any).rpasceneNodeHashCacheList?.find(
|
|
432
432
|
(item: { node: Node; id: string }) => item.id === id,
|
|
433
433
|
)?.node;
|
|
434
434
|
}
|
|
@@ -446,13 +446,13 @@ export function generateId(numberId: number) {
|
|
|
446
446
|
|
|
447
447
|
export function setGenerateHashOnWindow() {
|
|
448
448
|
if (typeof window !== 'undefined') {
|
|
449
|
-
(window as any).
|
|
449
|
+
(window as any).rpasceneGenerateHash = rpasceneGenerateHash;
|
|
450
450
|
}
|
|
451
451
|
}
|
|
452
452
|
|
|
453
|
-
export function
|
|
453
|
+
export function setRpasceneVisibleRectOnWindow() {
|
|
454
454
|
if (typeof window !== 'undefined') {
|
|
455
|
-
(window as any).
|
|
455
|
+
(window as any).rpasceneVisibleRect = elementRect;
|
|
456
456
|
}
|
|
457
457
|
}
|
|
458
458
|
|
|
@@ -21,7 +21,7 @@ import {
|
|
|
21
21
|
getRect,
|
|
22
22
|
getTopDocument,
|
|
23
23
|
logger,
|
|
24
|
-
|
|
24
|
+
rpasceneGenerateHash,
|
|
25
25
|
setDebugMode,
|
|
26
26
|
} from './util';
|
|
27
27
|
import { postWindowMessage, onWindowMessage } from './cs_postmessage';
|
|
@@ -83,7 +83,7 @@ export function collectElementInfo(
|
|
|
83
83
|
const attributes = getNodeAttributes(node, currentWindow);
|
|
84
84
|
let valueContent =
|
|
85
85
|
attributes.value || attributes.placeholder || node.textContent || '';
|
|
86
|
-
const nodeHashId =
|
|
86
|
+
const nodeHashId = rpasceneGenerateHash(node, valueContent, rect);
|
|
87
87
|
const tagName = (node as HTMLElement).tagName.toLowerCase();
|
|
88
88
|
if ((node as HTMLElement).tagName.toLowerCase() === 'select') {
|
|
89
89
|
// Get the selected option using the selectedIndex property
|
|
@@ -138,7 +138,7 @@ export function collectElementInfo(
|
|
|
138
138
|
const attributes = getNodeAttributes(node, currentWindow);
|
|
139
139
|
const pseudo = getPseudoElementContent(node, currentWindow);
|
|
140
140
|
const content = node.innerText || pseudo.before || pseudo.after || '';
|
|
141
|
-
const nodeHashId =
|
|
141
|
+
const nodeHashId = rpasceneGenerateHash(node, content, rect);
|
|
142
142
|
const elementInfo: WebElementInfo = {
|
|
143
143
|
id: nodeHashId,
|
|
144
144
|
indexId: indexId++,
|
|
@@ -163,7 +163,7 @@ export function collectElementInfo(
|
|
|
163
163
|
|
|
164
164
|
if (isImgElement(node, currentWindow)) {
|
|
165
165
|
const attributes = getNodeAttributes(node, currentWindow);
|
|
166
|
-
const nodeHashId =
|
|
166
|
+
const nodeHashId = rpasceneGenerateHash(node, '', rect);
|
|
167
167
|
const elementInfo: WebElementInfo = {
|
|
168
168
|
id: nodeHashId,
|
|
169
169
|
indexId: indexId++,
|
|
@@ -201,7 +201,7 @@ export function collectElementInfo(
|
|
|
201
201
|
if (!text.trim() && attributeKeys.length === 0) {
|
|
202
202
|
return null;
|
|
203
203
|
}
|
|
204
|
-
const nodeHashId =
|
|
204
|
+
const nodeHashId = rpasceneGenerateHash(node, text, rect);
|
|
205
205
|
const elementInfo: WebElementInfo = {
|
|
206
206
|
id: nodeHashId,
|
|
207
207
|
indexId: indexId++,
|
|
@@ -228,7 +228,7 @@ export function collectElementInfo(
|
|
|
228
228
|
const attributes = getNodeAttributes(node, currentWindow);
|
|
229
229
|
const pseudo = getPseudoElementContent(node, currentWindow);
|
|
230
230
|
const content = node.innerText || pseudo.before || pseudo.after || '';
|
|
231
|
-
const nodeHashId =
|
|
231
|
+
const nodeHashId = rpasceneGenerateHash(node, content, rect);
|
|
232
232
|
const elementInfo: WebElementInfo = {
|
|
233
233
|
id: nodeHashId,
|
|
234
234
|
indexId: indexId++,
|
|
@@ -255,7 +255,7 @@ export function collectElementInfo(
|
|
|
255
255
|
// 视为容器
|
|
256
256
|
if (isContainerElement(node, currentWindow) || isContainer) {
|
|
257
257
|
const attributes = getNodeAttributes(node, currentWindow);
|
|
258
|
-
const nodeHashId =
|
|
258
|
+
const nodeHashId = rpasceneGenerateHash(node, '', rect);
|
|
259
259
|
const elementInfo: WebElementInfo = {
|
|
260
260
|
id: nodeHashId,
|
|
261
261
|
nodeHashId,
|
package/src/logger.ts
CHANGED
|
@@ -2,10 +2,10 @@ import fs from 'node:fs';
|
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import util from 'node:util';
|
|
4
4
|
import debug from 'debug';
|
|
5
|
-
import {
|
|
5
|
+
import { getRpasceneRunSubDir } from './common';
|
|
6
6
|
import { ifInNode } from './utils';
|
|
7
7
|
|
|
8
|
-
const topicPrefix = '
|
|
8
|
+
const topicPrefix = 'rpascene';
|
|
9
9
|
// Map to store file streams
|
|
10
10
|
const logStreams = new Map<string, fs.WriteStream>();
|
|
11
11
|
// Map to store debug instances
|
|
@@ -16,7 +16,7 @@ function getLogStream(topic: string): fs.WriteStream {
|
|
|
16
16
|
const topicFileName = topic.replace(/:/g, '-');
|
|
17
17
|
if (!logStreams.has(topicFileName)) {
|
|
18
18
|
const logFile = path.join(
|
|
19
|
-
|
|
19
|
+
getRpasceneRunSubDir('log'),
|
|
20
20
|
`${topicFileName}.log`,
|
|
21
21
|
);
|
|
22
22
|
const stream = fs.createWriteStream(logFile, { flags: 'a' });
|
package/src/node/fs.ts
CHANGED
|
@@ -27,14 +27,14 @@ export function getRunningPkgInfo(dir?: string): PkgInfo | null {
|
|
|
27
27
|
if (pkgDir && pkgJsonFile) {
|
|
28
28
|
const { name, version } = JSON.parse(readFileSync(pkgJsonFile, 'utf-8'));
|
|
29
29
|
pkgCacheMap[dirToCheck] = {
|
|
30
|
-
name: name || '
|
|
30
|
+
name: name || 'rpascene-unknown-package-name',
|
|
31
31
|
version: version || '0.0.0',
|
|
32
32
|
dir: pkgDir,
|
|
33
33
|
};
|
|
34
34
|
return pkgCacheMap[dirToCheck];
|
|
35
35
|
}
|
|
36
36
|
return {
|
|
37
|
-
name: '
|
|
37
|
+
name: 'rpascene-unknown-package-name',
|
|
38
38
|
version: '0.0.0',
|
|
39
39
|
dir: dirToCheck,
|
|
40
40
|
};
|
|
@@ -75,10 +75,10 @@ export async function getExtraReturnLogic(tree = false) {
|
|
|
75
75
|
return null;
|
|
76
76
|
}
|
|
77
77
|
|
|
78
|
-
const elementInfosScriptContent = `;
|
|
78
|
+
const elementInfosScriptContent = `;rpascene_element_inspector.setNodeHashCacheListOnWindow();`;
|
|
79
79
|
|
|
80
80
|
if (tree) {
|
|
81
|
-
return `${elementInfosScriptContent};
|
|
81
|
+
return `${elementInfosScriptContent};rpascene_element_inspector.webExtractNodeTree()`;
|
|
82
82
|
}
|
|
83
|
-
return `${elementInfosScriptContent};
|
|
83
|
+
return `${elementInfosScriptContent};rpascene_element_inspector.webExtractTextWithPosition()`;
|
|
84
84
|
}
|
package/src/utils.ts
CHANGED
|
@@ -105,8 +105,8 @@ export async function repeat(
|
|
|
105
105
|
|
|
106
106
|
const REGEXP_LT = /</g;
|
|
107
107
|
const REGEXP_GT = />/g;
|
|
108
|
-
const REGEXP_LT_ESCAPE = '
|
|
109
|
-
const REGEXP_GT_ESCAPE = '
|
|
108
|
+
const REGEXP_LT_ESCAPE = '__rpascene_lt__';
|
|
109
|
+
const REGEXP_GT_ESCAPE = '__rpascene_gt__';
|
|
110
110
|
|
|
111
111
|
export const escapeScriptTag = (html: string) => {
|
|
112
112
|
return html
|