openclaw-cascade-plugin 1.0.16 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (13) hide show
  1. package/README.md +30 -28
  2. package/dist/grpc-client.d.ts +6 -1
  3. package/dist/grpc-client.d.ts.map +1 -1
  4. package/dist/grpc-client.js +14 -3
  5. package/dist/grpc-client.js.map +1 -1
  6. package/dist/test-utils/mocks.js +1 -1
  7. package/dist/test-utils/mocks.js.map +1 -1
  8. package/dist/tools/desktop-automation.d.ts.map +1 -1
  9. package/dist/tools/desktop-automation.js +295 -60
  10. package/dist/tools/desktop-automation.js.map +1 -1
  11. package/openclaw.plugin.json +1 -1
  12. package/package.json +1 -1
  13. package/proto/cascade.proto +74 -45
package/README.md CHANGED
@@ -1,32 +1,30 @@
1
- # OpenClaw Plugin for Cascade
1
+ # OpenClaw Cascade Plugin
2
2
 
3
- > Desktop automation meets conversational AI
4
-
5
- This package provides OpenClaw integration for the Cascade desktop automation platform. It is designed to work out-of-the-box with zero configuration required, lazily loading all dependencies (like Python) only when tools are actually used so it won't block your OpenClaw startup.
6
-
7
- ## Installation
3
+ This plugin connects OpenClaw to Cascade’s desktop + web automation stack. It loads quickly and only initializes heavier dependencies (Python, Playwright) when needed.
8
4
 
5
+ ## Install
9
6
  ```bash
10
7
  openclaw plugins install openclaw-cascade-plugin
11
8
  ```
12
9
 
13
- That's it! OpenClaw is now equipped with Cascade tools for desktop and web automation.
10
+ ## Quick Usage
11
+ ```bash
12
+ openclaw "Open Calculator"
13
+ openclaw "Click the 9 button"
14
+ openclaw "Type 8 + 3"
15
+ openclaw "Press Enter"
16
+ ```
14
17
 
15
18
  ## Configuration (Optional)
16
-
17
- The plugin defaults to connecting to a local Cascade gRPC endpoint (`localhost:50051`). No configuration in `~/.openclaw/config.json` is required.
18
-
19
- However, if you want to override the endpoint, enable A2A, or adjust screenshot behaviors, you can add an entry in your OpenClaw configuration file:
20
-
21
- ```json
19
+ The plugin defaults to `localhost:50051`. Add this to `~/.openclaw/openclaw.json` only if you need to override defaults:
20
+ ```json5
22
21
  {
23
- "plugins": {
24
- "entries": {
25
- "openclaw-cascade-plugin": {
26
- "enabled": true,
27
- "config": {
28
- "cascadeGrpcEndpoint": "192.168.1.100:50051",
29
- "firestoreProjectId": "your-project-id"
22
+ plugins: {
23
+ entries: {
24
+ openclaw-cascade-plugin: {
25
+ enabled: true,
26
+ config: {
27
+ cascadeGrpcEndpoint: "localhost:50051"
30
28
  }
31
29
  }
32
30
  }
@@ -34,16 +32,21 @@ However, if you want to override the endpoint, enable A2A, or adjust screenshot
34
32
  }
35
33
  ```
36
34
 
37
- If `mcp_server` is not found, set `cascadePythonModulePath` to your Cascade repo's `python/` directory.
35
+ Optional settings:
36
+ - `cascadePythonPath`
37
+ - `cascadePythonModulePath`
38
+ - `firestoreProjectId`, `firestoreCredentialsPath`
39
+ - `headless`, `actionTimeoutMs`
40
+ - `screenshotMode`, `screenshotDir`
41
+ - `enableA2A`, `allowedAgents`, `requireAgentConfirmation`
38
42
 
39
- If you previously installed `cascade` (v1.0.0), uninstall it and reinstall the latest version:
40
- ```bash
41
- openclaw plugins uninstall cascade
42
- openclaw plugins install openclaw-cascade-plugin@1.0.3
43
- ```
43
+ ## Tool Coverage
44
+ - **Automation (21)**: UIA3 click/type/hover/focus/scroll/wait + toggle, expand/collapse, select, range value, send keys, window state, move/resize. These actions also power web automation via `platform_source: WEB`.
45
+ - **A2A (3)**: Explorer/Worker/Orchestrator (opt-in).
44
46
 
45
- ## Development
47
+ For full details, see `../docs/openclaw-integration.md`.
46
48
 
49
+ ## Development
47
50
  ```bash
48
51
  npm install
49
52
  npm test
@@ -51,5 +54,4 @@ npm run build
51
54
  ```
52
55
 
53
56
  ## License
54
-
55
57
  MIT
package/dist/grpc-client.d.ts CHANGED
@@ -10,7 +10,12 @@ export declare class CascadeGrpcClient {
10
10
  stop(): void;
11
11
  startApp(appName: string): Promise<any>;
12
12
  resetState(): Promise<any>;
13
- performAction(actionType: string, selector: any, textPayload?: string): Promise<any>;
13
+ performAction(actionType: string, selector: any, payload?: {
14
+ text?: string;
15
+ number?: number;
16
+ json_payload?: string;
17
+ text_entry_mode?: string;
18
+ }): Promise<any>;
14
19
  getSemanticTree(): Promise<any>;
15
20
  getScreenshot(): Promise<any>;
16
21
  }
package/dist/grpc-client.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"grpc-client.d.ts","sourceRoot":"","sources":["../src/grpc-client.ts"],"names":[],"mappings":"AAOA,qBAAa,iBAAiB;IAMhB,OAAO,CAAC,QAAQ;IAL5B,OAAO,CAAC,aAAa,CAAM;IAC3B,OAAO,CAAC,gBAAgB,CAAM;IAC9B,OAAO,CAAC,YAAY,CAAM;IAC1B,OAAO,CAAC,aAAa,CAAS;gBAEV,QAAQ,EAAE,MAAM;IAE9B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA8B5B,WAAW,IAAI,OAAO;IAItB,IAAI,IAAI,IAAI;IAQN,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IASvC,UAAU,IAAI,OAAO,CAAC,GAAG,CAAC;IAU1B,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAkBpF,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC;IAU/B,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC;CAgBpC"}
1
+ {"version":3,"file":"grpc-client.d.ts","sourceRoot":"","sources":["../src/grpc-client.ts"],"names":[],"mappings":"AAOA,qBAAa,iBAAiB;IAMhB,OAAO,CAAC,QAAQ;IAL5B,OAAO,CAAC,aAAa,CAAM;IAC3B,OAAO,CAAC,gBAAgB,CAAM;IAC9B,OAAO,CAAC,YAAY,CAAM;IAC1B,OAAO,CAAC,aAAa,CAAS;gBAEV,QAAQ,EAAE,MAAM;IAE9B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA8B5B,WAAW,IAAI,OAAO;IAItB,IAAI,IAAI,IAAI;IAQN,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IASvC,UAAU,IAAI,OAAO,CAAC,GAAG,CAAC;IAU1B,aAAa,CACjB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,GAAG,EACb,OAAO,CAAC,EAAE;QACR,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,GACA,OAAO,CAAC,GAAG,CAAC;IA4BT,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC;IAU/B,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC;CAgBpC"}
package/dist/grpc-client.js CHANGED
@@ -104,13 +104,24 @@ class CascadeGrpcClient {
104
104
  });
105
105
  }
106
106
  // --- AutomationService ---
107
- async performAction(actionType, selector, textPayload) {
107
+ async performAction(actionType, selector, payload) {
108
108
  const action = {
109
109
  action_type: actionType,
110
110
  selector: selector
111
111
  };
112
- if (textPayload !== undefined) {
113
- action.text = textPayload;
112
+ if (payload) {
113
+ if (payload.text !== undefined) {
114
+ action.text = payload.text;
115
+ }
116
+ else if (payload.number !== undefined) {
117
+ action.number = payload.number;
118
+ }
119
+ else if (payload.json_payload !== undefined) {
120
+ action.json_payload = payload.json_payload;
121
+ }
122
+ if (payload.text_entry_mode !== undefined) {
123
+ action.text_entry_mode = payload.text_entry_mode;
124
+ }
114
125
  }
115
126
  return new Promise((resolve, reject) => {
116
127
  this.automationClient.PerformAction(action, (error, response) => {
package/dist/grpc-client.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"grpc-client.js","sourceRoot":"","sources":["../src/grpc-client.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAsC;AACtC,gEAAkD;AAClD,+BAA4B;AAE5B,+BAA+B;AAC/B,MAAM,UAAU,GAAG,IAAA,WAAI,EAAC,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;AAEnE,MAAa,iBAAiB;IAM5B,YAAoB,QAAgB;QAAhB,aAAQ,GAAR,QAAQ,CAAQ;QAF5B,kBAAa,GAAG,KAAK,CAAC;IAES,CAAC;IAExC,KAAK,CAAC,KAAK;QACT,IAAI,CAAC;YACH,MAAM,iBAAiB,GAAG,WAAW,CAAC,QAAQ,CAAC,UAAU,EAAE;gBACzD,QAAQ,EAAE,IAAI;gBACd,KAAK,EAAE,MAAM;gBACb,KAAK,EAAE,MAAM;gBACb,QAAQ,EAAE,IAAI;gBACd,MAAM,EAAE,IAAI;aACb,CAAC,CAAC;YAEH,MAAM,eAAe,GAAG,IAAI,CAAC,qBAAqB,CAAC,iBAAiB,CAAC,CAAC;YACtE,MAAM,OAAO,GAAQ,eAAe,CAAC,OAAO,CAAC;YAE7C,6EAA6E;YAC7E,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC;YACtD,MAAM,OAAO,GAAG;gBACd,iCAAiC,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,6BAA6B;aACnF,CAAC;YAEF,IAAI,CAAC,aAAa,GAAG,IAAI,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;YACrF,IAAI,CAAC,gBAAgB,GAAG,IAAI,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;YAC3F,IAAI,CAAC,YAAY,GAAG,IAAI,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;YAEnF,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC5B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC,CAAC;YAC1D,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,WAAW;QACT,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,aAAa;YAAE,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QACnD,IAAI,IAAI,CAAC,gBAAgB;YAAE,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QACzD,IAAI,IAAI,CAAC,YAAY;YAAE,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QACjD,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;IAC7B,CAAC;IAED,yBAAyB;IACzB,KAAK,CAAC,QAAQ,CAAC,OAAe;QAC5B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,CAAC,KAAU,EAAE,QAAa,EAAE,EAAE;gBAC/E,IAAI,KAAK;oBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;;oBACpB,OAAO,CAAC,QAAQ,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU;QACd,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,KAAU,EAAE,QAAa,EAAE,EAAE;gBAC9D,IAAI,KAAK;oBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;;oBACpB,OAAO,CAAC,QAAQ,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,4BAA4B;IAC5B,KAAK,CAAC,aAAa,CAAC,UAAkB,EAAE,QAAa,EAAE,WAAoB;QACzE,MAAM,MAAM,GAAQ;YAClB,WAAW,EAAE,UAAU;YACvB,QAAQ,EAAE,QAAQ;SACnB,CAAC;QAEF,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,CAAC,IAAI,GAAG,WAAW,CAAC;QAC5B,CAAC;QAED,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,KAAU,EAAE,QAAa,EAAE,EAAE;gBACxE,IAAI,KAAK;oBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;;oBACpB,OAAO,CAAC,QAAQ,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC,KAAU,EAAE,QAAa,EAAE,EAAE;gBACtE,IAAI,KAAK;oBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;;oBACpB,OAAO,CAAC,QAAQ,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,wBAAwB;IACxB,KAAK,CAAC,aAAa;QACjB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,EAAE,EAAE,CAAC,KAAU,EAAE,QAAa,EAAE,EAAE;gBACtE,IAAI,KAAK;oBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;qBACpB,CAAC;oBACJ,uFAAuF;oBACvF,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBACzE,OAAO,CAAC;wBACN,KAAK,EAAE,QAAQ;wBACf,MAAM,EAAE,QAAQ,CAAC,MAAM;wBACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;qBACtB,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAjHD,8CAiHC"}
1
+ {"version":3,"file":"grpc-client.js","sourceRoot":"","sources":["../src/grpc-client.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAsC;AACtC,gEAAkD;AAClD,+BAA4B;AAE5B,+BAA+B;AAC/B,MAAM,UAAU,GAAG,IAAA,WAAI,EAAC,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;AAEnE,MAAa,iBAAiB;IAM5B,YAAoB,QAAgB;QAAhB,aAAQ,GAAR,QAAQ,CAAQ;QAF5B,kBAAa,GAAG,KAAK,CAAC;IAES,CAAC;IAExC,KAAK,CAAC,KAAK;QACT,IAAI,CAAC;YACH,MAAM,iBAAiB,GAAG,WAAW,CAAC,QAAQ,CAAC,UAAU,EAAE;gBACzD,QAAQ,EAAE,IAAI;gBACd,KAAK,EAAE,MAAM;gBACb,KAAK,EAAE,MAAM;gBACb,QAAQ,EAAE,IAAI;gBACd,MAAM,EAAE,IAAI;aACb,CAAC,CAAC;YAEH,MAAM,eAAe,GAAG,IAAI,CAAC,qBAAqB,CAAC,iBAAiB,CAAC,CAAC;YACtE,MAAM,OAAO,GAAQ,eAAe,CAAC,OAAO,CAAC;YAE7C,6EAA6E;YAC7E,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC;YACtD,MAAM,OAAO,GAAG;gBACd,iCAAiC,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,6BAA6B;aACnF,CAAC;YAEF,IAAI,CAAC,aAAa,GAAG,IAAI,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;YACrF,IAAI,CAAC,gBAAgB,GAAG,IAAI,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;YAC3F,IAAI,CAAC,YAAY,GAAG,IAAI,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;YAEnF,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC5B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC,CAAC;YAC1D,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,WAAW;QACT,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,aAAa;YAAE,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QACnD,IAAI,IAAI,CAAC,gBAAgB;YAAE,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QACzD,IAAI,IAAI,CAAC,YAAY;YAAE,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QACjD,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;IAC7B,CAAC;IAED,yBAAyB;IACzB,KAAK,CAAC,QAAQ,CAAC,OAAe;QAC5B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,CAAC,KAAU,EAAE,QAAa,EAAE,EAAE;gBAC/E,IAAI,KAAK;oBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;;oBACpB,OAAO,CAAC,QAAQ,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU;QACd,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,KAAU,EAAE,QAAa,EAAE,EAAE;gBAC9D,IAAI,KAAK;oBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;;oBACpB,OAAO,CAAC,QAAQ,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,4BAA4B;IAC5B,KAAK,CAAC,aAAa,CACjB,UAAkB,EAClB,QAAa,EACb,OAKC;QAED,MAAM,MAAM,GAAQ;YAClB,WAAW,EAAE,UAAU;YACvB,QAAQ,EAAE,QAAQ;SACnB,CAAC;QAEF,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,CAAC;iBAAM,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBACxC,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YACjC,CAAC;iBAAM,IAAI,OAAO,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;gBAC9C,MAAM,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;YAC7C,CAAC;YAED,IAAI,OAAO,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;gBAC1C,MAAM,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;YACnD,CAAC;QACH,CAAC;QAED,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,KAAU,EAAE,QAAa,EAAE,EAAE;gBACxE,IAAI,KAAK;oBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;;oBACpB,OAAO,CAAC,QAAQ,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC,KAAU,EAAE,QAAa,EAAE,EAAE;gBACtE,IAAI,KAAK;oBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;;oBACpB,OAAO,CAAC,QAAQ,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,wBAAwB;IACxB,KAAK,CAAC,aAAa;QACjB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,EAAE,EAAE,CAAC,KAAU,EAAE,QAAa,EAAE,EAAE;gBACtE,IAAI,KAAK;oBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;qBACpB,CAAC;oBACJ,uFAAuF;oBACvF,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBACzE,OAAO,CAAC;wBACN,KAAK,EAAE,QAAQ;wBACf,MAAM,EAAE,QAAQ,CAAC,MAAM;wBACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;qBACtB,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AApID,8CAoIC"}
package/dist/test-utils/mocks.js CHANGED
@@ -38,7 +38,7 @@ class MockCascadeGrpcClient {
38
38
  }
39
39
  registerMockTool(name, handler) {
40
40
  if (name === 'click_element' || name === 'type_text' || name === 'hover_element' || name === 'focus_element' || name === 'scroll_element' || name === 'wait_visible') {
41
- this.performAction.mockImplementation(async (_type, selector, text) => handler({ selector, text }));
41
+ this.performAction.mockImplementation(async (_type, selector, payload) => handler({ selector, payload }));
42
42
  }
43
43
  else if (name === 'get_semantic_tree') {
44
44
  this.getSemanticTree.mockImplementation(async () => handler());
package/dist/test-utils/mocks.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"mocks.js","sourceRoot":"","sources":["../../src/test-utils/mocks.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAsBH,wDAqBC;AAzCD,mCAAsC;AAoBtC,SAAgB,sBAAsB;IACpC,MAAM,MAAM,GAAG,IAAI,qBAAY,EAAE,CAAC;IAClC,MAAM,MAAM,GAAG,IAAI,qBAAY,EAAE,CAAC;IAElC,OAAO;QACL,MAAM;QACN,MAAM;QACN,KAAK,EAAE;YACL,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE;SACjB;QACD,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE;QACb,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE;QACf,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE;QACf,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE;QACf,GAAG,EAAE,KAAK;QACV,SAAS,EAAE,IAAI;QACf,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE;QACrB,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE;QAChB,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE;QACd,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE;KAChB,CAAC;AACJ,CAAC;AAED,MAAa,qBAAqB;IAAlC;QACS,kBAAa,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;QAC1B,oBAAe,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;QAC5B,kBAAa,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;QAC1B,aAAQ,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;QACrB,eAAU,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;QACvB,gBAAW,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IAwBvD,CAAC;IAtBC,gBAAgB,CAAC,IAAY,EAAE,OAAiB;QAC9C,IAAI,IAAI,KAAK,eAAe,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,eAAe,IAAI,IAAI,KAAK,eAAe,IAAI,IAAI,KAAK,gBAAgB,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;YACrK,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACtG,CAAC;aAAM,IAAI,IAAI,KAAK,mBAAmB,EAAE,CAAC;YACxC,IAAI,CAAC,eAAe,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QACjE,CAAC;aAAM,IAAI,IAAI,KAAK,gBAAgB,EAAE,CAAC;YACrC,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;aAAM,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YAChC,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACtF,CAAC;IACH,CAAC;IAED,aAAa,CAAC,KAAY;QACxB,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAC5C,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC;IAED,uBAAuB;QACrB,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAC1C,CAAC;CACF;AA9BD,sDA8BC;AAED,MAAa,aAAa;IAA1B;QACU,SAAI,GAAG,IAAI,GAAG,EAAe,CAAC;IA2BxC,CAAC;IAzBC,WAAW,CAAC,IAAY,EAAE,IAAS;QACjC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC5B,CAAC;IAED,UAAU,CAAC,IAAY;QACrB,OAAO;YACL,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;gBAChB,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;qBAClC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;qBACvC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;oBACtB,EAAE,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE;oBACxB,IAAI,EAAE,GAAG,EAAE,CAAC,KAAK;iBAClB,CAAC,CAAC;aACN,CAAC;YACF,GAAG,EAAE,CAAC,EAAU,EAAE,EAAE,CAAC,CAAC;gBACpB,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;oBAChB,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,EAAE,EAAE,CAAC;oBACtC,IAAI,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,EAAE,EAAE,CAAC;iBAC3C,CAAC;gBACF,GAAG,EAAE,KAAK,EAAE,IAAS,EAAE,EAAE;oBACvB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;gBACvC,CAAC;aACF,CAAC;SACH,CAAC;IACJ,CAAC;CACF;AA5BD,sCA4BC"}
1
+ {"version":3,"file":"mocks.js","sourceRoot":"","sources":["../../src/test-utils/mocks.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAsBH,wDAqBC;AAzCD,mCAAsC;AAoBtC,SAAgB,sBAAsB;IACpC,MAAM,MAAM,GAAG,IAAI,qBAAY,EAAE,CAAC;IAClC,MAAM,MAAM,GAAG,IAAI,qBAAY,EAAE,CAAC;IAElC,OAAO;QACL,MAAM;QACN,MAAM;QACN,KAAK,EAAE;YACL,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE;SACjB;QACD,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE;QACb,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE;QACf,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE;QACf,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE;QACf,GAAG,EAAE,KAAK;QACV,SAAS,EAAE,IAAI;QACf,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE;QACrB,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE;QAChB,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE;QACd,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE;KAChB,CAAC;AACJ,CAAC;AAED,MAAa,qBAAqB;IAAlC;QACS,kBAAa,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;QAC1B,oBAAe,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;QAC5B,kBAAa,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;QAC1B,aAAQ,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;QACrB,eAAU,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;QACvB,gBAAW,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IAwBvD,CAAC;IAtBC,gBAAgB,CAAC,IAAY,EAAE,OAAiB;QAC9C,IAAI,IAAI,KAAK,eAAe,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,eAAe,IAAI,IAAI,KAAK,eAAe,IAAI,IAAI,KAAK,gBAAgB,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;YACrK,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAC5G,CAAC;aAAM,IAAI,IAAI,KAAK,mBAAmB,EAAE,CAAC;YACxC,IAAI,CAAC,eAAe,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QACjE,CAAC;aAAM,IAAI,IAAI,KAAK,gBAAgB,EAAE,CAAC;YACrC,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;aAAM,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YAChC,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACtF,CAAC;IACH,CAAC;IAED,aAAa,CAAC,KAAY;QACxB,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAC5C,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC;IAED,uBAAuB;QACrB,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAC1C,CAAC;CACF;AA9BD,sDA8BC;AAED,MAAa,aAAa;IAA1B;QACU,SAAI,GAAG,IAAI,GAAG,EAAe,CAAC;IA2BxC,CAAC;IAzBC,WAAW,CAAC,IAAY,EAAE,IAAS;QACjC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC5B,CAAC;IAED,UAAU,CAAC,IAAY;QACrB,OAAO;YACL,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;gBAChB,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;qBAClC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;qBACvC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;oBACtB,EAAE,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE;oBACxB,IAAI,EAAE,GAAG,EAAE,CAAC,KAAK;iBAClB,CAAC,CAAC;aACN,CAAC;YACF,GAAG,EAAE,CAAC,EAAU,EAAE,EAAE,CAAC,CAAC;gBACpB,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;oBAChB,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,EAAE,EAAE,CAAC;oBACtC,IAAI,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,EAAE,EAAE,CAAC;iBAC3C,CAAC;gBACF,GAAG,EAAE,KAAK,EAAE,IAAS,EAAE,EAAE;oBACvB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;gBACvC,CAAC;aACF,CAAC;SACH,CAAC;IACJ,CAAC;CACF;AA5BD,sCA4BC"}
package/dist/tools/desktop-automation.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"desktop-automation.d.ts","sourceRoot":"","sources":["../../src/tools/desktop-automation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAgB,mBAAmB,EAAE,MAAM,UAAU,CAAC;AAQ7D,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,YAAY,EACtB,SAAS,EAAE,MAAM,OAAO,CAAC,iBAAiB,CAAC,EAC3C,MAAM,CAAC,EAAE,mBAAmB,GAC3B,IAAI,CAqQN"}
1
+ {"version":3,"file":"desktop-automation.d.ts","sourceRoot":"","sources":["../../src/tools/desktop-automation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAgB,mBAAmB,EAAE,MAAM,UAAU,CAAC;AAQ7D,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,YAAY,EACtB,SAAS,EAAE,MAAM,OAAO,CAAC,iBAAiB,CAAC,EAC3C,MAAM,CAAC,EAAE,mBAAmB,GAC3B,IAAI,CAkfN"}
package/dist/tools/desktop-automation.js CHANGED
@@ -18,23 +18,39 @@ function registerDesktopTools(registry, getClient, config) {
18
18
  control_type: args.control_type,
19
19
  path: args.path,
20
20
  index: args.index,
21
- text_hint: args.text_hint
21
+ text_hint: args.text_hint,
22
+ element_id: args.element_id,
23
+ automation_id: args.automation_id,
24
+ class_name: args.class_name,
25
+ framework_id: args.framework_id,
26
+ help_text: args.help_text
22
27
  };
23
28
  };
29
+ const selectorProperties = {
30
+ platform_source: { type: 'string', enum: ['WINDOWS', 'JAVA', 'WEB'], description: 'Platform type' },
31
+ id: { type: 'string', description: 'Legacy automation ID' },
32
+ name: { type: 'string' },
33
+ control_type: { type: 'string' },
34
+ path: { type: 'array', items: { type: 'string' } },
35
+ index: { type: 'integer' },
36
+ text_hint: { type: 'string' },
37
+ element_id: { type: 'string', description: 'Runtime element ID' },
38
+ automation_id: { type: 'string', description: 'Automation ID' },
39
+ class_name: { type: 'string' },
40
+ framework_id: { type: 'string' },
41
+ help_text: { type: 'string' }
42
+ };
43
+ const selectorSchema = {
44
+ type: 'object',
45
+ properties: selectorProperties,
46
+ required: ['platform_source']
47
+ };
24
48
  registry.register({
25
49
  name: 'cascade_click_element',
26
50
  description: 'Click on a UI element on the desktop or in an application',
27
51
  inputSchema: {
28
52
  type: 'object',
29
- properties: {
30
- platform_source: { type: 'string', enum: ['WINDOWS', 'JAVA', 'WEB'], description: 'Platform type' },
31
- id: { type: 'string' },
32
- name: { type: 'string' },
33
- control_type: { type: 'string' },
34
- path: { type: 'array', items: { type: 'string' } },
35
- index: { type: 'integer' },
36
- text_hint: { type: 'string' }
37
- },
53
+ properties: selectorProperties,
38
54
  required: ['platform_source']
39
55
  },
40
56
  handler: async (args) => {
@@ -55,20 +71,9 @@ function registerDesktopTools(registry, getClient, config) {
55
71
  inputSchema: {
56
72
  type: 'object',
57
73
  properties: {
58
- selector: {
59
- type: 'object',
60
- properties: {
61
- platform_source: { type: 'string', enum: ['WINDOWS', 'JAVA', 'WEB'] },
62
- id: { type: 'string' },
63
- name: { type: 'string' },
64
- control_type: { type: 'string' },
65
- path: { type: 'array', items: { type: 'string' } },
66
- index: { type: 'integer' },
67
- text_hint: { type: 'string' }
68
- },
69
- required: ['platform_source']
70
- },
71
- text: { type: 'string', description: 'Text to type' }
74
+ selector: selectorSchema,
75
+ text: { type: 'string', description: 'Text to type' },
76
+ text_entry_mode: { type: 'string', enum: ['APPEND', 'REPLACE'] }
72
77
  },
73
78
  required: ['selector', 'text']
74
79
  },
@@ -78,7 +83,10 @@ function registerDesktopTools(registry, getClient, config) {
78
83
  return (0, response_helpers_1.errorResponse)('selector is required');
79
84
  if (!args.text)
80
85
  return (0, response_helpers_1.errorResponse)('text is required');
81
- const result = await (await getClient()).performAction('TYPE_TEXT', args.selector, args.text);
86
+ const result = await (await getClient()).performAction('TYPE_TEXT', args.selector, {
87
+ text: args.text,
88
+ text_entry_mode: args.text_entry_mode || 'APPEND'
89
+ });
82
90
  return (0, response_helpers_1.formatSuccess)(result);
83
91
  }
84
92
  catch (error) {
@@ -163,19 +171,7 @@ function registerDesktopTools(registry, getClient, config) {
163
171
  inputSchema: {
164
172
  type: 'object',
165
173
  properties: {
166
- selector: {
167
- type: 'object',
168
- properties: {
169
- platform_source: { type: 'string', enum: ['WINDOWS', 'JAVA', 'WEB'] },
170
- id: { type: 'string' },
171
- name: { type: 'string' },
172
- control_type: { type: 'string' },
173
- path: { type: 'array', items: { type: 'string' } },
174
- index: { type: 'integer' },
175
- text_hint: { type: 'string' }
176
- },
177
- required: ['platform_source']
178
- }
174
+ selector: selectorSchema
179
175
  },
180
176
  required: ['selector']
181
177
  },
@@ -195,13 +191,7 @@ function registerDesktopTools(registry, getClient, config) {
195
191
  inputSchema: {
196
192
  type: 'object',
197
193
  properties: {
198
- selector: {
199
- type: 'object',
200
- properties: {
201
- platform_source: { type: 'string', enum: ['WINDOWS', 'JAVA', 'WEB'] }
202
- },
203
- required: ['platform_source']
204
- }
194
+ selector: selectorSchema
205
195
  },
206
196
  required: ['selector']
207
197
  },
@@ -221,19 +211,16 @@ function registerDesktopTools(registry, getClient, config) {
221
211
  inputSchema: {
222
212
  type: 'object',
223
213
  properties: {
224
- selector: {
225
- type: 'object',
226
- properties: {
227
- platform_source: { type: 'string', enum: ['WINDOWS', 'JAVA', 'WEB'] }
228
- },
229
- required: ['platform_source']
230
- }
214
+ selector: selectorSchema,
215
+ amount: { type: 'number', description: 'Scroll delta (positive=down, negative=up)' }
231
216
  },
232
217
  required: ['selector']
233
218
  },
234
219
  handler: async (args) => {
235
220
  try {
236
- const result = await (await getClient()).performAction('SCROLL', args.selector);
221
+ const result = await (await getClient()).performAction('SCROLL', args.selector, {
222
+ number: args.amount
223
+ });
237
224
  return (0, response_helpers_1.formatSuccess)(result);
238
225
  }
239
226
  catch (error) {
@@ -247,13 +234,7 @@ function registerDesktopTools(registry, getClient, config) {
247
234
  inputSchema: {
248
235
  type: 'object',
249
236
  properties: {
250
- selector: {
251
- type: 'object',
252
- properties: {
253
- platform_source: { type: 'string', enum: ['WINDOWS', 'JAVA', 'WEB'] }
254
- },
255
- required: ['platform_source']
256
- }
237
+ selector: selectorSchema
257
238
  },
258
239
  required: ['selector']
259
240
  },
@@ -267,5 +248,259 @@ function registerDesktopTools(registry, getClient, config) {
267
248
  }
268
249
  }
269
250
  });
251
+ registry.register({
252
+ name: 'cascade_toggle_element',
253
+ description: 'Toggle a UI element (checkbox, switch, etc.)',
254
+ inputSchema: {
255
+ type: 'object',
256
+ properties: {
257
+ selector: selectorSchema
258
+ },
259
+ required: ['selector']
260
+ },
261
+ handler: async (args) => {
262
+ try {
263
+ const result = await (await getClient()).performAction('TOGGLE', args.selector);
264
+ return (0, response_helpers_1.formatSuccess)(result);
265
+ }
266
+ catch (error) {
267
+ return (0, response_helpers_1.errorResponse)(error instanceof Error ? error.message : 'Failed to toggle');
268
+ }
269
+ }
270
+ });
271
+ registry.register({
272
+ name: 'cascade_expand_element',
273
+ description: 'Expand a UI element (tree item, accordion, etc.)',
274
+ inputSchema: {
275
+ type: 'object',
276
+ properties: {
277
+ selector: selectorSchema
278
+ },
279
+ required: ['selector']
280
+ },
281
+ handler: async (args) => {
282
+ try {
283
+ const result = await (await getClient()).performAction('EXPAND', args.selector);
284
+ return (0, response_helpers_1.formatSuccess)(result);
285
+ }
286
+ catch (error) {
287
+ return (0, response_helpers_1.errorResponse)(error instanceof Error ? error.message : 'Failed to expand');
288
+ }
289
+ }
290
+ });
291
+ registry.register({
292
+ name: 'cascade_collapse_element',
293
+ description: 'Collapse a UI element (tree item, accordion, etc.)',
294
+ inputSchema: {
295
+ type: 'object',
296
+ properties: {
297
+ selector: selectorSchema
298
+ },
299
+ required: ['selector']
300
+ },
301
+ handler: async (args) => {
302
+ try {
303
+ const result = await (await getClient()).performAction('COLLAPSE', args.selector);
304
+ return (0, response_helpers_1.formatSuccess)(result);
305
+ }
306
+ catch (error) {
307
+ return (0, response_helpers_1.errorResponse)(error instanceof Error ? error.message : 'Failed to collapse');
308
+ }
309
+ }
310
+ });
311
+ registry.register({
312
+ name: 'cascade_select_element',
313
+ description: 'Select a UI element (list item, tab, etc.)',
314
+ inputSchema: {
315
+ type: 'object',
316
+ properties: {
317
+ selector: selectorSchema
318
+ },
319
+ required: ['selector']
320
+ },
321
+ handler: async (args) => {
322
+ try {
323
+ const result = await (await getClient()).performAction('SELECT', args.selector);
324
+ return (0, response_helpers_1.formatSuccess)(result);
325
+ }
326
+ catch (error) {
327
+ return (0, response_helpers_1.errorResponse)(error instanceof Error ? error.message : 'Failed to select');
328
+ }
329
+ }
330
+ });
331
+ registry.register({
332
+ name: 'cascade_set_range_value',
333
+ description: 'Set a range value (slider, spinner, etc.)',
334
+ inputSchema: {
335
+ type: 'object',
336
+ properties: {
337
+ selector: selectorSchema,
338
+ value: { type: 'number', description: 'Target numeric value' }
339
+ },
340
+ required: ['selector', 'value']
341
+ },
342
+ handler: async (args) => {
343
+ try {
344
+ const result = await (await getClient()).performAction('SET_RANGE_VALUE', args.selector, {
345
+ number: args.value
346
+ });
347
+ return (0, response_helpers_1.formatSuccess)(result);
348
+ }
349
+ catch (error) {
350
+ return (0, response_helpers_1.errorResponse)(error instanceof Error ? error.message : 'Failed to set range value');
351
+ }
352
+ }
353
+ });
354
+ registry.register({
355
+ name: 'cascade_send_keys',
356
+ description: 'Send a key chord or sequence to a focused element',
357
+ inputSchema: {
358
+ type: 'object',
359
+ properties: {
360
+ selector: selectorSchema,
361
+ keys: { type: 'string', description: 'Keys like ENTER, CTRL+S, ALT+F4' }
362
+ },
363
+ required: ['selector', 'keys']
364
+ },
365
+ handler: async (args) => {
366
+ try {
367
+ const result = await (await getClient()).performAction('SEND_KEYS', args.selector, {
368
+ text: args.keys
369
+ });
370
+ return (0, response_helpers_1.formatSuccess)(result);
371
+ }
372
+ catch (error) {
373
+ return (0, response_helpers_1.errorResponse)(error instanceof Error ? error.message : 'Failed to send keys');
374
+ }
375
+ }
376
+ });
377
+ registry.register({
378
+ name: 'cascade_window_minimize',
379
+ description: 'Minimize a window element',
380
+ inputSchema: {
381
+ type: 'object',
382
+ properties: {
383
+ selector: selectorSchema
384
+ },
385
+ required: ['selector']
386
+ },
387
+ handler: async (args) => {
388
+ try {
389
+ const result = await (await getClient()).performAction('WINDOW_MINIMIZE', args.selector);
390
+ return (0, response_helpers_1.formatSuccess)(result);
391
+ }
392
+ catch (error) {
393
+ return (0, response_helpers_1.errorResponse)(error instanceof Error ? error.message : 'Failed to minimize window');
394
+ }
395
+ }
396
+ });
397
+ registry.register({
398
+ name: 'cascade_window_maximize',
399
+ description: 'Maximize a window element',
400
+ inputSchema: {
401
+ type: 'object',
402
+ properties: {
403
+ selector: selectorSchema
404
+ },
405
+ required: ['selector']
406
+ },
407
+ handler: async (args) => {
408
+ try {
409
+ const result = await (await getClient()).performAction('WINDOW_MAXIMIZE', args.selector);
410
+ return (0, response_helpers_1.formatSuccess)(result);
411
+ }
412
+ catch (error) {
413
+ return (0, response_helpers_1.errorResponse)(error instanceof Error ? error.message : 'Failed to maximize window');
414
+ }
415
+ }
416
+ });
417
+ registry.register({
418
+ name: 'cascade_window_restore',
419
+ description: 'Restore a window element',
420
+ inputSchema: {
421
+ type: 'object',
422
+ properties: {
423
+ selector: selectorSchema
424
+ },
425
+ required: ['selector']
426
+ },
427
+ handler: async (args) => {
428
+ try {
429
+ const result = await (await getClient()).performAction('WINDOW_RESTORE', args.selector);
430
+ return (0, response_helpers_1.formatSuccess)(result);
431
+ }
432
+ catch (error) {
433
+ return (0, response_helpers_1.errorResponse)(error instanceof Error ? error.message : 'Failed to restore window');
434
+ }
435
+ }
436
+ });
437
+ registry.register({
438
+ name: 'cascade_window_close',
439
+ description: 'Close a window element',
440
+ inputSchema: {
441
+ type: 'object',
442
+ properties: {
443
+ selector: selectorSchema
444
+ },
445
+ required: ['selector']
446
+ },
447
+ handler: async (args) => {
448
+ try {
449
+ const result = await (await getClient()).performAction('WINDOW_CLOSE', args.selector);
450
+ return (0, response_helpers_1.formatSuccess)(result);
451
+ }
452
+ catch (error) {
453
+ return (0, response_helpers_1.errorResponse)(error instanceof Error ? error.message : 'Failed to close window');
454
+ }
455
+ }
456
+ });
457
+ registry.register({
458
+ name: 'cascade_move_window',
459
+ description: 'Move a window element by coordinates',
460
+ inputSchema: {
461
+ type: 'object',
462
+ properties: {
463
+ selector: selectorSchema,
464
+ x: { type: 'number', description: 'Target X coordinate' },
465
+ y: { type: 'number', description: 'Target Y coordinate' }
466
+ },
467
+ required: ['selector', 'x', 'y']
468
+ },
469
+ handler: async (args) => {
470
+ try {
471
+ const result = await (await getClient()).performAction('MOVE', args.selector, {
472
+ json_payload: JSON.stringify({ x: args.x, y: args.y })
473
+ });
474
+ return (0, response_helpers_1.formatSuccess)(result);
475
+ }
476
+ catch (error) {
477
+ return (0, response_helpers_1.errorResponse)(error instanceof Error ? error.message : 'Failed to move window');
478
+ }
479
+ }
480
+ });
481
+ registry.register({
482
+ name: 'cascade_resize_window',
483
+ description: 'Resize a window element by width/height',
484
+ inputSchema: {
485
+ type: 'object',
486
+ properties: {
487
+ selector: selectorSchema,
488
+ width: { type: 'number', description: 'Target width' },
489
+ height: { type: 'number', description: 'Target height' }
490
+ },
491
+ required: ['selector', 'width', 'height']
492
+ },
493
+ handler: async (args) => {
494
+ try {
495
+ const result = await (await getClient()).performAction('RESIZE', args.selector, {
496
+ json_payload: JSON.stringify({ width: args.width, height: args.height })
497
+ });
498
+ return (0, response_helpers_1.formatSuccess)(result);
499
+ }
500
+ catch (error) {
501
+ return (0, response_helpers_1.errorResponse)(error instanceof Error ? error.message : 'Failed to resize window');
502
+ }
503
+ }
504
+ });
270
505
  }
271
506
  //# sourceMappingURL=desktop-automation.js.map
package/dist/tools/desktop-automation.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"desktop-automation.js","sourceRoot":"","sources":["../../src/tools/desktop-automation.ts"],"names":[],"mappings":";;AAUA,oDAyQC;AAhRD,yDAAiF;AACjF,0CAA+C;AAC/C,+BAA4B;AAC5B,2BAA6B;AAE7B,MAAM,cAAc,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAEvC,SAAgB,oBAAoB,CAClC,QAAsB,EACtB,SAA2C,EAC3C,MAA4B;IAE5B,MAAM,cAAc,GAAG,MAAM,EAAE,cAAc,IAAI,MAAM,CAAC;IACxD,MAAM,aAAa,GAAG,MAAM,EAAE,aAAa,IAAI,IAAA,WAAI,EAAC,IAAA,YAAO,GAAE,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;IAE3F,6BAA6B;IAC7B,MAAM,eAAe,GAAG,CAAC,IAAS,EAAE,EAAE;QACpC,OAAO;YACL,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC;IACJ,CAAC,CAAC;IAEF,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,uBAAuB;QAC7B,WAAW,EAAE,2DAA2D;QACxE,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,eAAe,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,WAAW,EAAE,eAAe,EAAE;gBACnG,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACtB,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACxB,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAChC,IAAI,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;gBAClD,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC1B,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC9B;YACD,QAAQ,EAAE,CAAC,iBAAiB,CAAC;SAC9B;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC,eAAe;oBAAE,OAAO,IAAA,gCAAa,EAAC,6BAA6B,CAAC,CAAC;gBAC/E,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,OAAO,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;gBACvF,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,mBAAmB;QACzB,WAAW,EAAE,6BAA6B;QAC1C,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE;oBACR,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE;wBACV,eAAe,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE;wBACrE,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;wBACtB,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;wBACxB,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;wBAChC,IAAI,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;wBAClD,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;wBAC1B,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC9B;oBACD,QAAQ,EAAE,CAAC,iBAAiB,CAAC;iBAC9B;gBACD,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE;aACtD;YACD,QAAQ,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC;SAC/B;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC,QAAQ;oBAAE,OAAO,IAAA,gCAAa,EAAC,sBAAsB,CAAC,CAAC;gBACjE,IAAI,CAAC,IAAI,CAAC,IAAI;oBAAE,OAAO,IAAA,gCAAa,EAAC,kBAAkB,CAAC,CAAC;gBACzD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC9F,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC;YACvF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,2BAA2B;QACjC,WAAW,EAAE,8BAA8B;QAC3C,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE;SAClD;QACD,OAAO,EAAE,KAAK,IAA2B,EAAE;YACzC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,eAAe,EAAE,CAAC;gBAC3D,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC;YACtF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,wBAAwB;QAC9B,WAAW,EAAE,+CAA+C;QAC5D,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE;SAClD;QACD,OAAO,EAAE,KAAK,IAA2B,EAAE;YACzC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,EAAE,CAAC;gBACzD,IAAI,CAAC,MAAM,CAAC,KAAK;oBAAE,OAAO,IAAA,gCAAa,EAAC,wBAAwB,CAAC,CAAC;gBAElE,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,MAAM,CAAC;gBAC7D,MAAM,WAAW,GAAG,cAAc,KAAK,OAAO,IAAI,CAAC,cAAc,KAAK,MAAM,IAAI,SAAS,GAAG,cAAc,CAAC,CAAC;gBAE5G,IAAI,WAAW,EAAE,CAAC;oBAChB,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,0CAA0C,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;gBAChI,CAAC;gBAED,MAAM,IAAA,gBAAK,EAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAChD,MAAM,QAAQ,GAAG,cAAc,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC;gBAChD,MAAM,QAAQ,GAAG,IAAA,WAAI,EAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;gBAC/C,MAAM,IAAA,oBAAS,EAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;gBAE/D,OAAO,IAAA,gCAAa,EAAC;oBACnB,OAAO,EAAE,uBAAuB,QAAQ,EAAE;oBAC1C,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,SAAS,EAAE,QAAQ;iBACpB,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC;YAC5F,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,mBAAmB;QACzB,WAAW,EAAE,8BAA8B;QAC3C,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC7B;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC,QAAQ;oBAAE,OAAO,IAAA,gCAAa,EAAC,sBAAsB,CAAC,CAAC;gBACjE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACjE,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC;YACvF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,uBAAuB;QAC7B,WAAW,EAAE,yBAAyB;QACtC,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE;oBACR,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE;wBACV,eAAe,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE;wBACrE,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;wBACtB,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;wBACxB,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;wBAChC,IAAI,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;wBAClD,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;wBAC1B,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC9B;oBACD,QAAQ,EAAE,CAAC,iBAAiB,CAAC;iBAC9B;aACF;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC/E,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;YACnF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,uBAAuB;QAC7B,WAAW,EAAE,uBAAuB;QACpC,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE;oBACR,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE;wBACV,eAAe,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE;qBACtE;oBACD,QAAQ,EAAE,CAAC,iBAAiB,CAAC;iBAC9B;aACF;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC/E,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;YACnF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,wBAAwB;QAC9B,WAAW,EAAE,qBAAqB;QAClC,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE;oBACR,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE;wBACV,eAAe,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE;qBACtE;oBACD,QAAQ,EAAE,CAAC,iBAAiB,CAAC;iBAC9B;aACF;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChF,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC;YACpF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,sBAAsB;QAC5B,WAAW,EAAE,yCAAyC;QACtD,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE;oBACR,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE;wBACV,eAAe,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE;qBACtE;oBACD,QAAQ,EAAE,CAAC,iBAAiB,CAAC;iBAC9B;aACF;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtF,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;YAClF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}
1
+ {"version":3,"file":"desktop-automation.js","sourceRoot":"","sources":["../../src/tools/desktop-automation.ts"],"names":[],"mappings":";;AAUA,oDAsfC;AA7fD,yDAAiF;AACjF,0CAA+C;AAC/C,+BAA4B;AAC5B,2BAA6B;AAE7B,MAAM,cAAc,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAEvC,SAAgB,oBAAoB,CAClC,QAAsB,EACtB,SAA2C,EAC3C,MAA4B;IAE5B,MAAM,cAAc,GAAG,MAAM,EAAE,cAAc,IAAI,MAAM,CAAC;IACxD,MAAM,aAAa,GAAG,MAAM,EAAE,aAAa,IAAI,IAAA,WAAI,EAAC,IAAA,YAAO,GAAE,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;IAE3F,6BAA6B;IAC7B,MAAM,eAAe,GAAG,CAAC,IAAS,EAAE,EAAE;QACpC,OAAO;YACL,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,kBAAkB,GAAG;QACzB,eAAe,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,WAAW,EAAE,eAAe,EAAE;QACnG,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,sBAAsB,EAAE;QAC3D,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QACxB,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAChC,IAAI,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;QAClD,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;QAC1B,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC7B,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,oBAAoB,EAAE;QACjE,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE;QAC/D,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC9B,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAChC,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;KAC9B,CAAC;IAEF,MAAM,cAAc,GAAG;QACrB,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE,kBAAkB;QAC9B,QAAQ,EAAE,CAAC,iBAAiB,CAAC;KAC9B,CAAC;IAEF,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,uBAAuB;QAC7B,WAAW,EAAE,2DAA2D;QACxE,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,kBAAkB;YAC9B,QAAQ,EAAE,CAAC,iBAAiB,CAAC;SAC9B;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC,eAAe;oBAAE,OAAO,IAAA,gCAAa,EAAC,6BAA6B,CAAC,CAAC;gBAC/E,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,OAAO,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;gBACvF,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,mBAAmB;QACzB,WAAW,EAAE,6BAA6B;QAC1C,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,cAAc;gBACxB,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE;gBACrD,eAAe,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE;aACjE;YACD,QAAQ,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC;SAC/B;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC,QAAQ;oBAAE,OAAO,IAAA,gCAAa,EAAC,sBAAsB,CAAC,CAAC;gBACjE,IAAI,CAAC,IAAI,CAAC,IAAI;oBAAE,OAAO,IAAA,gCAAa,EAAC,kBAAkB,CAAC,CAAC;gBACzD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,QAAQ,EAAE;oBACjF,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,eAAe,EAAE,IAAI,CAAC,eAAe,IAAI,QAAQ;iBAClD,CAAC,CAAC;gBACH,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC;YACvF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,2BAA2B;QACjC,WAAW,EAAE,8BAA8B;QAC3C,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE;SAClD;QACD,OAAO,EAAE,KAAK,IAA2B,EAAE;YACzC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,eAAe,EAAE,CAAC;gBAC3D,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC;YACtF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,wBAAwB;QAC9B,WAAW,EAAE,+CAA+C;QAC5D,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE;SAClD;QACD,OAAO,EAAE,KAAK,IAA2B,EAAE;YACzC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,EAAE,CAAC;gBACzD,IAAI,CAAC,MAAM,CAAC,KAAK;oBAAE,OAAO,IAAA,gCAAa,EAAC,wBAAwB,CAAC,CAAC;gBAElE,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,MAAM,CAAC;gBAC7D,MAAM,WAAW,GAAG,cAAc,KAAK,OAAO,IAAI,CAAC,cAAc,KAAK,MAAM,IAAI,SAAS,GAAG,cAAc,CAAC,CAAC;gBAE5G,IAAI,WAAW,EAAE,CAAC;oBAChB,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,0CAA0C,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;gBAChI,CAAC;gBAED,MAAM,IAAA,gBAAK,EAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAChD,MAAM,QAAQ,GAAG,cAAc,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC;gBAChD,MAAM,QAAQ,GAAG,IAAA,WAAI,EAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;gBAC/C,MAAM,IAAA,oBAAS,EAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;gBAE/D,OAAO,IAAA,gCAAa,EAAC;oBACnB,OAAO,EAAE,uBAAuB,QAAQ,EAAE;oBAC1C,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,SAAS,EAAE,QAAQ;iBACpB,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC;YAC5F,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,mBAAmB;QACzB,WAAW,EAAE,8BAA8B;QAC3C,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC7B;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC,QAAQ;oBAAE,OAAO,IAAA,gCAAa,EAAC,sBAAsB,CAAC,CAAC;gBACjE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACjE,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC;YACvF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,uBAAuB;QAC7B,WAAW,EAAE,yBAAyB;QACtC,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,cAAc;aACzB;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC/E,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;YACnF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,uBAAuB;QAC7B,WAAW,EAAE,uBAAuB;QACpC,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,cAAc;aACzB;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC/E,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;YACnF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,wBAAwB;QAC9B,WAAW,EAAE,qBAAqB;QAClC,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,cAAc;gBACxB,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,2CAA2C,EAAE;aACrF;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE;oBAC9E,MAAM,EAAE,IAAI,CAAC,MAAM;iBACpB,CAAC,CAAC;gBACH,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC;YACpF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,sBAAsB;QAC5B,WAAW,EAAE,yCAAyC;QACtD,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,cAAc;aACzB;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtF,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;YAClF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,wBAAwB;QAC9B,WAAW,EAAE,8CAA8C;QAC3D,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,cAAc;aACzB;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChF,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC;YACpF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,wBAAwB;QAC9B,WAAW,EAAE,kDAAkD;QAC/D,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,cAAc;aACzB;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChF,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC;YACpF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,0BAA0B;QAChC,WAAW,EAAE,oDAAoD;QACjE,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,cAAc;aACzB;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAClF,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC;YACtF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,wBAAwB;QAC9B,WAAW,EAAE,4CAA4C;QACzD,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,cAAc;aACzB;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChF,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC;YACpF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,yBAAyB;QAC/B,WAAW,EAAE,2CAA2C;QACxD,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,cAAc;gBACxB,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,sBAAsB,EAAE;aAC/D;YACD,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC;SAChC;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,iBAAiB,EAAE,IAAI,CAAC,QAAQ,EAAE;oBACvF,MAAM,EAAE,IAAI,CAAC,KAAK;iBACnB,CAAC,CAAC;gBACH,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC;YAC7F,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,mBAAmB;QACzB,WAAW,EAAE,mDAAmD;QAChE,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,cAAc;gBACxB,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,iCAAiC,EAAE;aACzE;YACD,QAAQ,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC;SAC/B;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,QAAQ,EAAE;oBACjF,IAAI,EAAE,IAAI,CAAC,IAAI;iBAChB,CAAC,CAAC;gBACH,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC;YACvF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,yBAAyB;QAC/B,WAAW,EAAE,2BAA2B;QACxC,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,cAAc;aACzB;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,iBAAiB,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACzF,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC;YAC7F,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,yBAAyB;QAC/B,WAAW,EAAE,2BAA2B;QACxC,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,cAAc;aACzB;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,iBAAiB,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACzF,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC;YAC7F,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,wBAAwB;QAC9B,WAAW,EAAE,0BAA0B;QACvC,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,cAAc;aACzB;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,gBAAgB,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACxF,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC;YAC5F,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,sBAAsB;QAC5B,WAAW,EAAE,wBAAwB;QACrC,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,cAAc;aACzB;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtF,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC;YAC1F,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,qBAAqB;QAC3B,WAAW,EAAE,sCAAsC;QACnD,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,cAAc;gBACxB,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qBAAqB,EAAE;gBACzD,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qBAAqB,EAAE;aAC1D;YACD,QAAQ,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC;SACjC;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE;oBAC5E,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC;iBACvD,CAAC,CAAC;gBACH,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC;YACzF,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,CAAC;QAChB,IAAI,EAAE,uBAAuB;QAC7B,WAAW,EAAE,yCAAyC;QACtD,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,cAAc;gBACxB,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE;gBACtD,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE;aACzD;YACD,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,QAAQ,CAAC;SAC1C;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAyB,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE;oBAC9E,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;iBACzE,CAAC,CAAC;gBACH,OAAO,IAAA,gCAAa,EAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,IAAA,gCAAa,EAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}
package/openclaw.plugin.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "id": "openclaw-cascade-plugin",
3
3
  "name": "Cascade Desktop Automation",
4
4
  "description": "Control Windows desktop apps and web browsers through OpenClaw. Provides desktop automation, web automation with Playwright, and skill learning capabilities.",
5
- "version": "1.0.14",
5
+ "version": "1.1.0",
6
6
  "configSchema": {
7
7
  "type": "object",
8
8
  "additionalProperties": false,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "openclaw-cascade-plugin",
3
- "version": "1.0.16",
3
+ "version": "1.1.0",
4
4
  "description": "Desktop automation plugin for OpenClaw powered by Cascade",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
package/proto/cascade.proto CHANGED
@@ -42,42 +42,52 @@ message SemanticTree {
42
42
  repeated UIElement elements = 1;
43
43
  }
44
44
 
45
- message UIElement {
46
- // Stable ID within a session; provider is responsible for consistency.
47
- string id = 1;
48
- string name = 2;
49
- ControlType control_type = 3;
50
- NormalizedRectangle bounding_box = 4;
51
- string parent_id = 5;
52
- PlatformSource platform_source = 6;
53
- // Optional accessibility/automation hints that vary by platform.
54
- optional string aria_role = 7;
55
- optional string automation_id = 8;
56
- optional string value_text = 9;
57
- }
58
-
59
- message Selector {
60
- PlatformSource platform_source = 1;
61
- // Provider-specific path components; interpretation is platform-specific but normalized here.
62
- repeated string path = 2;
63
- // Optional filters to disambiguate elements.
64
- optional string id = 3;
65
- optional string name = 4;
66
- optional ControlType control_type = 5;
67
- optional int32 index = 6;
68
- optional string text_hint = 7;
69
- }
70
-
71
- message Action {
72
- ActionType action_type = 1;
73
- Selector selector = 2;
74
- // Optional payload for the action (text entry, numeric input, or structured JSON).
75
- oneof payload {
76
- string text = 3;
77
- double number = 4;
78
- string json_payload = 5;
79
- }
80
- }
45
+ message UIElement {
46
+ // Stable ID within a session; provider is responsible for consistency.
47
+ string id = 1;
48
+ string name = 2;
49
+ ControlType control_type = 3;
50
+ NormalizedRectangle bounding_box = 4;
51
+ string parent_id = 5;
52
+ PlatformSource platform_source = 6;
53
+ // Optional accessibility/automation hints that vary by platform.
54
+ optional string aria_role = 7;
55
+ optional string automation_id = 8;
56
+ optional string value_text = 9;
57
+ optional string class_name = 10;
58
+ optional string framework_id = 11;
59
+ optional string help_text = 12;
60
+ }
61
+
62
+ message Selector {
63
+ PlatformSource platform_source = 1;
64
+ // Provider-specific path components; interpretation is platform-specific but normalized here.
65
+ repeated string path = 2;
66
+ // Optional filters to disambiguate elements.
67
+ optional string id = 3;
68
+ optional string name = 4;
69
+ optional ControlType control_type = 5;
70
+ optional int32 index = 6;
71
+ optional string text_hint = 7;
72
+ optional string element_id = 8;
73
+ optional string automation_id = 9;
74
+ optional string class_name = 10;
75
+ optional string framework_id = 11;
76
+ optional string help_text = 12;
77
+ }
78
+
79
+ message Action {
80
+ ActionType action_type = 1;
81
+ Selector selector = 2;
82
+ // Optional payload for the action (text entry, numeric input, or structured JSON).
83
+ oneof payload {
84
+ string text = 3;
85
+ double number = 4;
86
+ string json_payload = 5;
87
+ }
88
+ TextEntryMode text_entry_mode = 6;
89
+ }
90
+
81
91
 
82
92
  message Screenshot {
83
93
  bytes image = 1;
@@ -98,15 +108,34 @@ message NormalizedRectangle {
98
108
  double height = 4;
99
109
  }
100
110
 
101
- enum ActionType {
102
- ACTION_TYPE_UNSPECIFIED = 0;
103
- CLICK = 1;
104
- TYPE_TEXT = 2;
105
- HOVER = 3;
106
- FOCUS = 4;
107
- SCROLL = 5;
108
- WAIT_VISIBLE = 6;
109
- }
111
+ enum ActionType {
112
+ ACTION_TYPE_UNSPECIFIED = 0;
113
+ CLICK = 1;
114
+ TYPE_TEXT = 2;
115
+ HOVER = 3;
116
+ FOCUS = 4;
117
+ SCROLL = 5;
118
+ WAIT_VISIBLE = 6;
119
+ TOGGLE = 7;
120
+ EXPAND = 8;
121
+ COLLAPSE = 9;
122
+ SELECT = 10;
123
+ SET_RANGE_VALUE = 11;
124
+ SEND_KEYS = 12;
125
+ WINDOW_MINIMIZE = 13;
126
+ WINDOW_MAXIMIZE = 14;
127
+ WINDOW_RESTORE = 15;
128
+ WINDOW_CLOSE = 16;
129
+ MOVE = 17;
130
+ RESIZE = 18;
131
+ }
132
+
133
+ enum TextEntryMode {
134
+ TEXT_ENTRY_MODE_UNSPECIFIED = 0;
135
+ APPEND = 1;
136
+ REPLACE = 2;
137
+ }
138
+
110
139
 
111
140
  enum PlatformSource {
112
141
  PLATFORM_SOURCE_UNSPECIFIED = 0;