@forsion/tangu-computer-use 0.5.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/CHANGELOG.md +223 -0
  2. package/LICENSE +26 -0
  3. package/LICENSE.upstream +21 -0
  4. package/README.md +169 -0
  5. package/UPSTREAM.md +277 -0
  6. package/check.mjs +369 -0
  7. package/icon.png +0 -0
  8. package/install.sh +112 -0
  9. package/main.js +541 -0
  10. package/manifest.json +47 -0
  11. package/native/linux/bridge-rs/Cargo.lock +1204 -0
  12. package/native/linux/bridge-rs/Cargo.toml +18 -0
  13. package/native/linux/bridge-rs/src/atspi.rs +640 -0
  14. package/native/linux/bridge-rs/src/error.rs +65 -0
  15. package/native/linux/bridge-rs/src/lib.rs +11 -0
  16. package/native/linux/bridge-rs/src/main.rs +1074 -0
  17. package/native/linux/bridge-rs/src/protocol.rs +49 -0
  18. package/native/linux/bridge-rs/src/state.rs +293 -0
  19. package/native/linux/bridge-rs/src/wayland.rs +89 -0
  20. package/native/linux/bridge-rs/src/x11.rs +909 -0
  21. package/native/linux/bridge-rs/tests/protocol_tests.rs +73 -0
  22. package/native/macos/agent_cursor.swift +184 -0
  23. package/native/macos/agent_cursor_motion.swift +252 -0
  24. package/native/macos/agent_cursor_tests.swift +128 -0
  25. package/native/macos/agent_highlight.swift +229 -0
  26. package/native/macos/agent_highlight_tests.swift +99 -0
  27. package/native/macos/bridge.swift +3852 -0
  28. package/native/macos/foreground_activity.swift +57 -0
  29. package/native/macos/foreground_activity_tests.swift +31 -0
  30. package/native/macos/live_stream.swift +289 -0
  31. package/native/windows/bridge-rs/Cargo.lock +396 -0
  32. package/native/windows/bridge-rs/Cargo.toml +30 -0
  33. package/native/windows/bridge-rs/src/capture.rs +518 -0
  34. package/native/windows/bridge-rs/src/error.rs +81 -0
  35. package/native/windows/bridge-rs/src/input.rs +501 -0
  36. package/native/windows/bridge-rs/src/lib.rs +16 -0
  37. package/native/windows/bridge-rs/src/main.rs +1683 -0
  38. package/native/windows/bridge-rs/src/protocol.rs +53 -0
  39. package/native/windows/bridge-rs/src/refs.rs +237 -0
  40. package/native/windows/bridge-rs/src/state.rs +55 -0
  41. package/native/windows/bridge-rs/src/uia.rs +1252 -0
  42. package/native/windows/bridge-rs/src/window.rs +718 -0
  43. package/native/windows/bridge-rs/tests/protocol_tests.rs +252 -0
  44. package/native/windows/bridge-rs/tests/refs_tests.rs +86 -0
  45. package/native/windows/bridge-rs/tests/state_tests.rs +33 -0
  46. package/package.json +74 -0
  47. package/prebuilt/linux/arm64/linux-bridge +0 -0
  48. package/prebuilt/linux/x64/linux-bridge +0 -0
  49. package/prebuilt/macos/arm64/bridge +0 -0
  50. package/prebuilt/macos/arm64/tangu-computer-use.app.json +10 -0
  51. package/prebuilt/macos/arm64/tangu-computer-use.app.zip +0 -0
  52. package/prebuilt/macos/x64/bridge +0 -0
  53. package/prebuilt/macos/x64/tangu-computer-use.app.json +10 -0
  54. package/prebuilt/macos/x64/tangu-computer-use.app.zip +0 -0
  55. package/prebuilt/windows/windows-bridge.exe +0 -0
  56. package/scripts/blind-click.check.mjs +128 -0
  57. package/scripts/build-native.mjs +302 -0
  58. package/scripts/build.mjs +84 -0
  59. package/scripts/calc-fixture.mjs +78 -0
  60. package/scripts/cli-exit.check.mjs +49 -0
  61. package/scripts/helper-path.check.mjs +82 -0
  62. package/scripts/helper-refresh.check.mjs +62 -0
  63. package/scripts/helper-signal.check.mjs +59 -0
  64. package/scripts/highlight.check.mjs +32 -0
  65. package/scripts/keychain-free-install.check.mjs +140 -0
  66. package/scripts/live-view.check.mjs +124 -0
  67. package/scripts/macos-bundle.d.mts +1 -0
  68. package/scripts/macos-bundle.mjs +119 -0
  69. package/scripts/make-signing-cert.sh +57 -0
  70. package/scripts/mini-foreground.check.mjs +11 -0
  71. package/scripts/no-foreground.check.mjs +81 -0
  72. package/scripts/overlay-visible.check.mjs +221 -0
  73. package/scripts/package-macos-app.mjs +39 -0
  74. package/scripts/permissions.check.mjs +199 -0
  75. package/scripts/platform-contract.check.mjs +30 -0
  76. package/scripts/setup-helper.mjs +154 -0
  77. package/scripts/tangu-computer-use.entitlements +10 -0
  78. package/scripts/verify-macos-bundles.mjs +25 -0
  79. package/scripts/verify-package.mjs +74 -0
  80. package/skills/computer-use/SKILL.md +110 -0
  81. package/src/foregroundNote.ts +43 -0
  82. package/src/helperState.ts +31 -0
  83. package/src/index.ts +63 -0
  84. package/src/onboarding.ts +198 -0
  85. package/src/pi-compat.ts +49 -0
  86. package/src/settings.ts +11 -0
  87. package/src/setup.ts +97 -0
  88. package/src/tools.ts +289 -0
  89. package/src/vendor/actions.ts +130 -0
  90. package/src/vendor/bridge.ts +2405 -0
  91. package/src/vendor/cdp.ts +658 -0
  92. package/src/vendor/config.ts +113 -0
  93. package/src/vendor/contract.ts +104 -0
  94. package/src/vendor/note.ts +195 -0
  95. package/src/vendor/outline.ts +651 -0
  96. package/src/vendor/output.ts +134 -0
  97. package/src/vendor/permissions.ts +111 -0
  98. package/src/vendor/platform/architecture.ts +23 -0
  99. package/src/vendor/platform/coerce.ts +16 -0
  100. package/src/vendor/platform/index.ts +59 -0
  101. package/src/vendor/platform/linux/backend.ts +186 -0
  102. package/src/vendor/platform/linux/helper.ts +238 -0
  103. package/src/vendor/platform/macos/backend.ts +131 -0
  104. package/src/vendor/platform/macos/browser.ts +110 -0
  105. package/src/vendor/platform/macos/helper-path.d.mts +9 -0
  106. package/src/vendor/platform/macos/helper-path.mjs +34 -0
  107. package/src/vendor/platform/macos/helper.ts +291 -0
  108. package/src/vendor/platform/macos/permissions.ts +146 -0
  109. package/src/vendor/platform/types.ts +222 -0
  110. package/src/vendor/platform/windows/backend.ts +142 -0
  111. package/src/vendor/platform/windows/helper.ts +140 -0
  112. package/src/vendor/root-selection.ts +25 -0
  113. package/src/vendor/runtime.ts +129 -0
  114. package/src/vendor/state.ts +146 -0
  115. package/src/vendor/view.ts +147 -0
  116. package/tangu-plugins/computer-use/dist/foregroundNote.js +19 -0
  117. package/tangu-plugins/computer-use/dist/index.js +5672 -0
  118. package/tangu-plugins/computer-use/tangu-plugin.json +9 -0
  119. package/tsconfig.json +22 -0
  120. package/types/tangu-agent.d.ts +142 -0
@@ -0,0 +1,252 @@
1
+ use serde_json::json;
2
+
3
+ /// The protocol types under test are expected to be exported from the library crate.
4
+ use windows_bridge::{protocol::PROTOCOL_VERSION, ErrorCode, ProtocolError, Request, Response};
5
+
6
+ // ---------------------------------------------------------------------------
7
+ // Request parsing
8
+ // ---------------------------------------------------------------------------
9
+
10
+ #[test]
11
+ fn test_parse_valid_request() {
12
+ let raw = json!({
13
+ "protocolVersion": PROTOCOL_VERSION,
14
+ "id": "req-001",
15
+ "cmd": "checkPermissions",
16
+ "args": {}
17
+ });
18
+ let req: Request = serde_json::from_value(raw).expect("valid request");
19
+ assert_eq!(req.protocol_version, PROTOCOL_VERSION);
20
+ assert_eq!(req.id, "req-001");
21
+ assert_eq!(req.cmd, "checkPermissions");
22
+ }
23
+
24
+ #[test]
25
+ fn test_parse_request_with_args() {
26
+ let raw = json!({
27
+ "protocolVersion": PROTOCOL_VERSION,
28
+ "id": "req-002",
29
+ "cmd": "listWindows",
30
+ "args": { "pid": 1234 }
31
+ });
32
+ let req: Request = serde_json::from_value(raw).expect("valid request with args");
33
+ assert_eq!(req.protocol_version, PROTOCOL_VERSION);
34
+ assert_eq!(req.id, "req-002");
35
+ assert_eq!(req.cmd, "listWindows");
36
+ assert!(req.args.is_object());
37
+ assert_eq!(req.args["pid"], 1234);
38
+ }
39
+
40
+ #[test]
41
+ fn test_parse_request_missing_id() {
42
+ let raw = json!({
43
+ "protocolVersion": PROTOCOL_VERSION,
44
+ "cmd": "checkPermissions",
45
+ "args": {}
46
+ });
47
+ let result: Result<Request, _> = serde_json::from_value(raw);
48
+ assert!(result.is_err(), "missing id should be rejected");
49
+ }
50
+
51
+ #[test]
52
+ fn test_parse_request_missing_cmd() {
53
+ let raw = json!({
54
+ "protocolVersion": PROTOCOL_VERSION,
55
+ "id": "req-003",
56
+ "args": {}
57
+ });
58
+ let result: Result<Request, _> = serde_json::from_value(raw);
59
+ assert!(result.is_err(), "missing cmd should be rejected");
60
+ }
61
+
62
+ #[test]
63
+ fn test_parse_request_wrong_types() {
64
+ // id must be a string
65
+ let raw = json!({
66
+ "protocolVersion": PROTOCOL_VERSION,
67
+ "id": 42,
68
+ "cmd": "test",
69
+ "args": {}
70
+ });
71
+ let result: Result<Request, _> = serde_json::from_value(raw);
72
+ assert!(result.is_err(), "non-string id should be rejected");
73
+ }
74
+
75
+ // ---------------------------------------------------------------------------
76
+ // Success response serialization
77
+ // ---------------------------------------------------------------------------
78
+
79
+ #[test]
80
+ fn test_serialize_success_response() {
81
+ let resp = Response::ok("req-001", json!({ "ready": true }));
82
+ let json_str = serde_json::to_string(&resp).expect("serialize success");
83
+ let parsed: serde_json::Value = serde_json::from_str(&json_str).unwrap();
84
+
85
+ assert_eq!(parsed["protocolVersion"], PROTOCOL_VERSION);
86
+ assert_eq!(parsed["id"], "req-001");
87
+ assert_eq!(parsed["ok"], true);
88
+ assert_eq!(parsed["result"]["ready"], true);
89
+ }
90
+
91
+ #[test]
92
+ fn test_serialize_success_with_null_result() {
93
+ let resp = Response::ok("req-002", json!(null));
94
+ let json_str = serde_json::to_string(&resp).expect("serialize success null");
95
+ let parsed: serde_json::Value = serde_json::from_str(&json_str).unwrap();
96
+
97
+ assert_eq!(parsed["protocolVersion"], PROTOCOL_VERSION);
98
+ assert_eq!(parsed["id"], "req-002");
99
+ assert_eq!(parsed["ok"], true);
100
+ assert_eq!(parsed["result"], serde_json::Value::Null);
101
+ }
102
+
103
+ // ---------------------------------------------------------------------------
104
+ // Error response serialization
105
+ // ---------------------------------------------------------------------------
106
+
107
+ #[test]
108
+ fn test_serialize_capability_deferred_error() {
109
+ let err = ProtocolError::new(
110
+ "Window ref-backed actions are deferred in PR #1. \
111
+ This PR supports window discovery, screenshots, state IDs, \
112
+ and read-only UIA element discovery.",
113
+ ErrorCode::CapabilityDeferred,
114
+ );
115
+ let resp = Response::err("req-003", err);
116
+ let json_str = serde_json::to_string(&resp).expect("serialize error");
117
+ let parsed: serde_json::Value = serde_json::from_str(&json_str).unwrap();
118
+
119
+ assert_eq!(parsed["protocolVersion"], PROTOCOL_VERSION);
120
+ assert_eq!(parsed["id"], "req-003");
121
+ assert_eq!(parsed["ok"], false);
122
+ assert_eq!(parsed["error"]["code"], "capability_deferred");
123
+ assert!(parsed["error"]["message"]
124
+ .as_str()
125
+ .unwrap()
126
+ .contains("deferred"));
127
+ }
128
+
129
+ #[test]
130
+ fn test_serialize_unsupported_command_error() {
131
+ let err = ProtocolError::new("Unknown command 'fooBar'", ErrorCode::UnsupportedCommand);
132
+ let resp = Response::err("req-004", err);
133
+ let json_str = serde_json::to_string(&resp).expect("serialize error");
134
+ let parsed: serde_json::Value = serde_json::from_str(&json_str).unwrap();
135
+
136
+ assert_eq!(parsed["protocolVersion"], PROTOCOL_VERSION);
137
+ assert_eq!(parsed["id"], "req-004");
138
+ assert_eq!(parsed["ok"], false);
139
+ assert_eq!(parsed["error"]["code"], "unsupported_command");
140
+ assert_eq!(parsed["error"]["message"], "Unknown command 'fooBar'");
141
+ }
142
+
143
+ #[test]
144
+ fn test_serialize_invalid_request_error() {
145
+ let err = ProtocolError::new(
146
+ "Request must be a valid JSON object with protocolVersion, id, and cmd",
147
+ ErrorCode::InvalidRequest,
148
+ );
149
+ let resp = Response::err("req-005", err);
150
+ let json_str = serde_json::to_string(&resp).expect("serialize error");
151
+ let parsed: serde_json::Value = serde_json::from_str(&json_str).unwrap();
152
+
153
+ assert_eq!(parsed["protocolVersion"], PROTOCOL_VERSION);
154
+ assert_eq!(parsed["id"], "req-005");
155
+ assert_eq!(parsed["ok"], false);
156
+ assert_eq!(parsed["error"]["code"], "invalid_request");
157
+ }
158
+
159
+ #[test]
160
+ fn test_serialize_internal_error() {
161
+ let err = ProtocolError::new("Unexpected internal failure", ErrorCode::InternalError);
162
+ let resp = Response::err("req-006", err);
163
+ let json_str = serde_json::to_string(&resp).expect("serialize error");
164
+ let parsed: serde_json::Value = serde_json::from_str(&json_str).unwrap();
165
+
166
+ assert_eq!(parsed["protocolVersion"], PROTOCOL_VERSION);
167
+ assert_eq!(parsed["id"], "req-006");
168
+ assert_eq!(parsed["ok"], false);
169
+ assert_eq!(parsed["error"]["code"], "internal_error");
170
+ }
171
+
172
+ #[test]
173
+ fn test_serialize_target_not_found_error() {
174
+ let err = ProtocolError::new("Window with id 42 not found", ErrorCode::TargetNotFound);
175
+ let resp = Response::err("req-007", err);
176
+ let json_str = serde_json::to_string(&resp).expect("serialize error");
177
+ let parsed: serde_json::Value = serde_json::from_str(&json_str).unwrap();
178
+
179
+ assert_eq!(parsed["protocolVersion"], PROTOCOL_VERSION);
180
+ assert_eq!(parsed["id"], "req-007");
181
+ assert_eq!(parsed["ok"], false);
182
+ assert_eq!(parsed["error"]["code"], "target_not_found");
183
+ }
184
+
185
+ // ---------------------------------------------------------------------------
186
+ // ErrorCode Display
187
+ // ---------------------------------------------------------------------------
188
+
189
+ #[test]
190
+ fn test_error_code_display() {
191
+ assert_eq!(
192
+ ErrorCode::CapabilityDeferred.to_string(),
193
+ "capability_deferred"
194
+ );
195
+ assert_eq!(
196
+ ErrorCode::UnsupportedCommand.to_string(),
197
+ "unsupported_command"
198
+ );
199
+ assert_eq!(ErrorCode::InvalidRequest.to_string(), "invalid_request");
200
+ assert_eq!(ErrorCode::TargetNotFound.to_string(), "target_not_found");
201
+ assert_eq!(ErrorCode::InternalError.to_string(), "internal_error");
202
+ }
203
+
204
+ // ---------------------------------------------------------------------------
205
+ // Integration: round-trip request + response
206
+ // ---------------------------------------------------------------------------
207
+
208
+ #[test]
209
+ fn test_round_trip_request_response() {
210
+ // Simulate: parse a request, produce a success response, serialize it
211
+ let raw = json!({
212
+ "protocolVersion": PROTOCOL_VERSION,
213
+ "id": "rt-001",
214
+ "cmd": "checkPermissions",
215
+ "args": {}
216
+ });
217
+ let req: Request = serde_json::from_value(raw).unwrap();
218
+ assert_eq!(req.cmd, "checkPermissions");
219
+
220
+ let resp = Response::ok(req.id.as_str(), json!({ "accessibility": false }));
221
+ let out = serde_json::to_string(&resp).unwrap();
222
+ let parsed: serde_json::Value = serde_json::from_str(&out).unwrap();
223
+ assert_eq!(parsed["protocolVersion"], PROTOCOL_VERSION);
224
+ assert_eq!(parsed["id"], "rt-001");
225
+ assert_eq!(parsed["ok"], true);
226
+ assert_eq!(parsed["result"]["accessibility"], false);
227
+ }
228
+
229
+ #[test]
230
+ fn test_round_trip_request_error() {
231
+ // Simulate: parse request, produce an error response, serialize it
232
+ let raw = json!({
233
+ "protocolVersion": PROTOCOL_VERSION,
234
+ "id": "rt-002",
235
+ "cmd": "mouseClick",
236
+ "args": {}
237
+ });
238
+ let req: Request = serde_json::from_value(raw).unwrap();
239
+ assert_eq!(req.cmd, "mouseClick");
240
+
241
+ let err = ProtocolError::new(
242
+ "Windows ref-backed actions are deferred in PR #1",
243
+ ErrorCode::CapabilityDeferred,
244
+ );
245
+ let resp = Response::err(req.id.as_str(), err);
246
+ let out = serde_json::to_string(&resp).unwrap();
247
+ let parsed: serde_json::Value = serde_json::from_str(&out).unwrap();
248
+ assert_eq!(parsed["protocolVersion"], PROTOCOL_VERSION);
249
+ assert_eq!(parsed["id"], "rt-002");
250
+ assert_eq!(parsed["ok"], false);
251
+ assert_eq!(parsed["error"]["code"], "capability_deferred");
252
+ }
@@ -0,0 +1,86 @@
1
+ use windows_bridge::refs::{NativeHandle, RefStore};
2
+
3
+ // ---------------------------------------------------------------------------
4
+ // First ref assertions
5
+ // ---------------------------------------------------------------------------
6
+
7
+ #[test]
8
+ fn test_first_window_ref_is_w1() {
9
+ let mut store = RefStore::new();
10
+ let wref = store.insert_window(NativeHandle::new(0x1234));
11
+ assert_eq!(wref.to_string(), "@w1");
12
+ }
13
+
14
+ #[test]
15
+ fn test_first_element_ref_is_e1() {
16
+ let mut store = RefStore::new();
17
+ let eref = store.insert_element(NativeHandle::new(0x5678));
18
+ assert_eq!(eref.to_string(), "@e1");
19
+ }
20
+
21
+ // ---------------------------------------------------------------------------
22
+ // Refs are state-scoped: independent stores reset counters
23
+ // ---------------------------------------------------------------------------
24
+
25
+ #[test]
26
+ fn test_refs_are_state_scoped() {
27
+ let mut store1 = RefStore::new();
28
+ let mut store2 = RefStore::new();
29
+
30
+ let w1 = store1.insert_window(NativeHandle::new(1));
31
+ let e1 = store1.insert_element(NativeHandle::new(2));
32
+ let w2 = store2.insert_window(NativeHandle::new(3));
33
+ let e2 = store2.insert_element(NativeHandle::new(4));
34
+
35
+ assert_eq!(w1.to_string(), "@w1");
36
+ assert_eq!(e1.to_string(), "@e1");
37
+ // Second store starts fresh at 1
38
+ assert_eq!(w2.to_string(), "@w1");
39
+ assert_eq!(e2.to_string(), "@e1");
40
+ }
41
+
42
+ // ---------------------------------------------------------------------------
43
+ // Monotonically increasing refs within the same store
44
+ // ---------------------------------------------------------------------------
45
+
46
+ #[test]
47
+ fn test_window_refs_increment() {
48
+ let mut store = RefStore::new();
49
+ let w1 = store.insert_window(NativeHandle::new(10));
50
+ let w2 = store.insert_window(NativeHandle::new(20));
51
+ let w3 = store.insert_window(NativeHandle::new(30));
52
+
53
+ assert_eq!(w1.to_string(), "@w1");
54
+ assert_eq!(w2.to_string(), "@w2");
55
+ assert_eq!(w3.to_string(), "@w3");
56
+ }
57
+
58
+ #[test]
59
+ fn test_element_refs_increment() {
60
+ let mut store = RefStore::new();
61
+ let e1 = store.insert_element(NativeHandle::new(100));
62
+ let e2 = store.insert_element(NativeHandle::new(200));
63
+
64
+ assert_eq!(e1.to_string(), "@e1");
65
+ assert_eq!(e2.to_string(), "@e2");
66
+ }
67
+
68
+ // ---------------------------------------------------------------------------
69
+ // Retrieve stored native handles by ref
70
+ // ---------------------------------------------------------------------------
71
+
72
+ #[test]
73
+ fn test_retrieve_window_handle() {
74
+ let mut store = RefStore::new();
75
+ let wref = store.insert_window(NativeHandle::new(0xABCD));
76
+ let retrieved = store.get_window(&wref);
77
+ assert_eq!(retrieved, Some(NativeHandle::new(0xABCD)));
78
+ }
79
+
80
+ #[test]
81
+ fn test_retrieve_element_handle() {
82
+ let mut store = RefStore::new();
83
+ let eref = store.insert_element(NativeHandle::new(0xDEAD));
84
+ let retrieved = store.get_element(&eref);
85
+ assert_eq!(retrieved, Some(NativeHandle::new(0xDEAD)));
86
+ }
@@ -0,0 +1,33 @@
1
+ use windows_bridge::state::StateId;
2
+
3
+ // ---------------------------------------------------------------------------
4
+ // Fresh state IDs
5
+ // ---------------------------------------------------------------------------
6
+
7
+ #[test]
8
+ fn test_two_fresh_state_ids_differ() {
9
+ let id1 = StateId::fresh("session");
10
+ let id2 = StateId::fresh("session");
11
+ assert_ne!(id1, id2, "two fresh state IDs must differ");
12
+ }
13
+
14
+ #[test]
15
+ fn test_fresh_state_id_contains_prefix() {
16
+ let id = StateId::fresh("test");
17
+ assert!(
18
+ id.starts_with("test-"),
19
+ "state ID should start with prefix followed by '-'"
20
+ );
21
+ }
22
+
23
+ #[test]
24
+ fn test_fresh_ids_are_unique_across_calls() {
25
+ let mut ids = Vec::new();
26
+ for _ in 0..100 {
27
+ ids.push(StateId::fresh("batch"));
28
+ }
29
+ let mut sorted = ids.clone();
30
+ sorted.sort();
31
+ sorted.dedup();
32
+ assert_eq!(sorted.len(), ids.len(), "all 100 fresh IDs must be unique");
33
+ }
package/package.json ADDED
@@ -0,0 +1,74 @@
1
+ {
2
+ "name": "@forsion/tangu-computer-use",
3
+ "version": "0.5.8",
4
+ "description": "Tangu engine plugin: let agents observe and control macOS, Windows, and Linux desktop apps. Fork of injaneity/pi-computer-use (MIT).",
5
+ "type": "module",
6
+ "license": "MIT",
7
+ "repository": {
8
+ "type": "git",
9
+ "url": "git+https://github.com/Changan-Su/Tangu-Computer-Use.git"
10
+ },
11
+ "publishConfig": {
12
+ "access": "public"
13
+ },
14
+ "keywords": [
15
+ "tangu-plugin",
16
+ "computer-use",
17
+ "macos",
18
+ "desktop-automation",
19
+ "accessibility"
20
+ ],
21
+ "engines": {
22
+ "node": ">=20"
23
+ },
24
+ "files": [
25
+ "tangu-plugins",
26
+ "skills",
27
+ "manifest.json",
28
+ "icon.png",
29
+ "main.js",
30
+ "check.mjs",
31
+ "install.sh",
32
+ "CHANGELOG.md",
33
+ "UPSTREAM.md",
34
+ "tsconfig.json",
35
+ "src",
36
+ "types",
37
+ "scripts",
38
+ "prebuilt",
39
+ "native/macos",
40
+ "native/windows",
41
+ "native/linux/bridge-rs",
42
+ "LICENSE",
43
+ "LICENSE.upstream",
44
+ "README.md"
45
+ ],
46
+ "scripts": {
47
+ "build": "node scripts/build.mjs && npm run typecheck",
48
+ "typecheck": "tsc --noEmit",
49
+ "check:no-foreground": "node scripts/no-foreground.check.mjs",
50
+ "check:helper-path": "node scripts/helper-path.check.mjs",
51
+ "check:highlight": "node scripts/highlight.check.mjs",
52
+ "check:permissions": "node scripts/permissions.check.mjs",
53
+ "check:plugin": "node check.mjs",
54
+ "check:platform": "node scripts/platform-contract.check.mjs",
55
+ "check:blindclick": "node scripts/blind-click.check.mjs",
56
+ "check:liveview": "node scripts/live-view.check.mjs",
57
+ "check": "npm run check:platform && npm run check:no-foreground && npm run check:helper-path && npm run check:highlight && npm run check:permissions && npm run check:plugin && npm run check:helper-refresh && npm run check:installer && npm run check:cli-exit",
58
+ "build:native": "node scripts/build-native.mjs",
59
+ "build:windows": "node scripts/build-native.mjs --platform windows",
60
+ "build:linux": "node scripts/build-native.mjs --platform linux",
61
+ "check:live": "npm run check:blindclick && npm run check:liveview && npm run check:overlay",
62
+ "check:overlay": "node scripts/overlay-visible.check.mjs",
63
+ "check:mini-foreground": "node scripts/mini-foreground.check.mjs",
64
+ "check:helper-refresh": "node scripts/helper-refresh.check.mjs",
65
+ "check:helper-signal": "node scripts/helper-signal.check.mjs",
66
+ "check:installer": "node scripts/keychain-free-install.check.mjs",
67
+ "check:cli-exit": "node scripts/cli-exit.check.mjs"
68
+ },
69
+ "devDependencies": {
70
+ "@types/node": "^22.10.0",
71
+ "esbuild": "^0.24.0",
72
+ "typescript": "^5.6.0"
73
+ }
74
+ }
Binary file
Binary file
@@ -0,0 +1,10 @@
1
+ {
2
+ "bundleId": "com.forsion.tangu-computer-use",
3
+ "version": "0.5.8",
4
+ "archiveSha256": "30aa8caad4f8a5425af2b72516b22e8947982ad42d7cb7eccc6a7bbebfb7c264",
5
+ "files": {
6
+ "Contents/MacOS/bridge": "e83fb3ee7bd4bbdcbe64876485602354c6881b5e7ee73306ef2fcdb716484063",
7
+ "Contents/Info.plist": "6089a47275854f04a74c57cd0f862de65781bfed323f9fba65e0768aa592518a",
8
+ "Contents/_CodeSignature/CodeResources": "6686de10a28a2fe11b36cbb86dcbacc827cfc4ea116b4dabf1845e5aee629e9b"
9
+ }
10
+ }
Binary file
@@ -0,0 +1,10 @@
1
+ {
2
+ "bundleId": "com.forsion.tangu-computer-use",
3
+ "version": "0.5.8",
4
+ "archiveSha256": "a8c677df28ee97c320d10319773662f4dad9c0f0d880c185291949837909c8be",
5
+ "files": {
6
+ "Contents/MacOS/bridge": "3e6f53eeec547db886b57b033e82ab9e9dc31164d0d613fbf60847243751255e",
7
+ "Contents/Info.plist": "6089a47275854f04a74c57cd0f862de65781bfed323f9fba65e0768aa592518a",
8
+ "Contents/_CodeSignature/CodeResources": "6686de10a28a2fe11b36cbb86dcbacc827cfc4ea116b4dabf1845e5aee629e9b"
9
+ }
10
+ }
@@ -0,0 +1,128 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * 「后台盲点」的真机仪器 —— 证明**坐标点击不抢前台**,而且真的落下去了。
4
+ *
5
+ * 为什么必须有它:症状是"点了就跳前台、虚拟鼠标不出现",而这条路上有三段各自都会静默失败的东西
6
+ * (needsForeground 判定 → helper 的 AX 命中测试 → AXPress 到底打没打中)。纸面推理只能验证想到的
7
+ * 那一段。这个脚本一次把三段都钉住:
8
+ *
9
+ * ① `performed.delivery === "ax"` —— 走的是 AX 按压,没发物理事件(所以不动真鼠标)
10
+ * ② 前台 App 前后不变 —— 没抢焦点
11
+ * ③ **计算器的显示值真的变了** —— 这一条最重要:前两条都能在"点击压根没生效"时照样成立。
12
+ * 没有 ③ 的话,"不抢前台"可以靠什么都不做来达成。
13
+ *
14
+ * 跑法:node scripts/blind-click.check.mjs
15
+ * 需要:macOS、helper 已装并授权(辅助功能 + 屏幕录制)。会短暂打开「计算器」,跑完关掉。
16
+ */
17
+ import assert from 'node:assert/strict'
18
+ import { connect } from 'node:net'
19
+ import { homedir } from 'node:os'
20
+ import { ensureDaemon, freshCalculator, killCalculator } from './calc-fixture.mjs'
21
+
22
+ const SOCK = `${homedir()}/Library/Caches/tangu-computer-use/bridge.sock`
23
+ const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
24
+
25
+ let seq = 0
26
+ /** 一次请求一条连接 —— helper 的协议是一行 JSON 进、一行 JSON 出,不必维持长连。 */
27
+ function ask(payload) {
28
+ return new Promise((resolve, reject) => {
29
+ const sock = connect(SOCK)
30
+ let buf = ''
31
+ const timer = setTimeout(() => { sock.destroy(); reject(new Error(`timeout: ${payload.cmd}`)) }, 20_000)
32
+ sock.on('error', (e) => { clearTimeout(timer); reject(e) })
33
+ sock.on('data', (d) => {
34
+ buf += d
35
+ const nl = buf.indexOf('\n')
36
+ if (nl < 0) return
37
+ clearTimeout(timer)
38
+ sock.end()
39
+ const msg = JSON.parse(buf.slice(0, nl))
40
+ if (!msg.ok) return reject(Object.assign(new Error(msg.error?.message || 'helper error'), { code: msg.error?.code }))
41
+ resolve(msg.result)
42
+ })
43
+ sock.write(`${JSON.stringify({ id: `bc_${++seq}`, ...payload })}\n`)
44
+ })
45
+ }
46
+
47
+ /** 前台是谁 —— 问 helper 自己(NSWorkspace),**别走 osascript**:System Events 要额外的
48
+ * 「自动化」授权,node 没有,只会 AppleEvent 超时(踩过)。 */
49
+ const frontmostApp = async () => `${(await ask({ cmd: 'getFrontmost' })).appName}`
50
+
51
+ /** 在 outline 里深度优先找第一个满足条件的节点。 */
52
+ function findNode(node, pred) {
53
+ if (!node) return null
54
+ if (pred(node)) return node
55
+ for (const child of node.children || []) {
56
+ const hit = findNode(child, pred)
57
+ if (hit) return hit
58
+ }
59
+ return null
60
+ }
61
+
62
+ const diag = await ensureDaemon(ask)
63
+ assert.equal(diag.protocolVersion, 11, `helper 协议应为 11,实测 ${diag.protocolVersion} —— 先重装 helper`)
64
+ assert.ok(diag.accessibility, '需要「辅助功能」权限')
65
+
66
+ try {
67
+ const before = await frontmostApp()
68
+ // 后台开一个全新的计算器(-g = 不抢前台);夹具会先等上一个仪器留下的实例真的退干净。
69
+ const { calc, win } = await freshCalculator(ask)
70
+
71
+ const look = await ask({ cmd: 'look', pid: calc.pid, windowId: win.windowId, includeImage: true })
72
+ // ⚠️outline 里的 rect 已经是**图像坐标**(不是屏幕坐标),act 的 x/y 要的正是它 —— 别再换算一遍。
73
+ assert.ok(
74
+ findNode(look.outline, (n) => n.role === 'AXButton' && (n.description === '5' || n.title === '5')),
75
+ '没在 outline 里找到数字键 5',
76
+ )
77
+
78
+ /** 计算器的读数:第一个有值的静态文本。数字前有个 LTR 标记,只留数字。 */
79
+ const displayIn = (outline) => String(findNode(outline, (n) => n.role === 'AXStaticText' && n.value)?.value ?? '').replace(/\D/g, '')
80
+ const digitAt = (n) => {
81
+ const node = findNode(look.outline, (x) => x.role === 'AXButton' && (x.description === n || x.title === n))
82
+ assert.ok(node, `没在 outline 里找到数字键 ${n}`)
83
+ // ⚠️outline 里的 rect 已经是**图像坐标**(不是屏幕坐标),act 的 x/y 要的正是它 —— 别再换算一遍。
84
+ return { x: node.rect.x + node.rect.w / 2, y: node.rect.y + node.rect.h / 2 }
85
+ }
86
+
87
+ /** 只给 x/y(**没有 ref**)—— 这正是 agent 看着截图"盲点"时走的那条路,也正是上游会直接判
88
+ * needsForeground、跳去 HID 前台的那条路。policy/delivery 照 TS 层在这种情况下会发的值。 */
89
+ const blindClick = async (digit) => {
90
+ const p = digitAt(digit)
91
+ const act = await ask({
92
+ cmd: 'act',
93
+ lookId: look.lookId,
94
+ pid: calc.pid,
95
+ policy: 'foreground',
96
+ delivery: 'hid',
97
+ cursorOverlay: true,
98
+ action: 'click',
99
+ target: { x: p.x, y: p.y },
100
+ params: { button: 'left', clickCount: 1 },
101
+ })
102
+ await sleep(350)
103
+ const fresh = await ask({ cmd: 'look', pid: calc.pid, windowId: win.windowId, includeImage: false })
104
+ return { act, display: displayIn(fresh.outline) }
105
+ }
106
+
107
+ // 连点两个**不同**的数字:断言只看"最后一位是不是刚点的那个",与计算器的起始状态无关
108
+ // (上一轮跑剩的数字会留在屏上 —— 拿 before≠after 当判据会被这个坑掉)。
109
+ const five = await blindClick('5')
110
+ const seven = await blindClick('7')
111
+ const after = await frontmostApp()
112
+
113
+ console.log(` performed = ${JSON.stringify(five.act.performed)}`)
114
+ console.log(` 前台: ${before} → ${after}`)
115
+ console.log(` 计算器显示: 点 5 → ${five.display} | 点 7 → ${seven.display}`)
116
+
117
+ for (const [label, r] of [['5', five], ['7', seven]]) {
118
+ assert.equal(r.act.performed?.delivery, 'ax', `点 ${label}:坐标点击应走后台 AX 按压(delivery=ax),不是物理事件`)
119
+ assert.equal(r.act.performed?.grounding, 'hit_test', `点 ${label}:应经由 AX 命中测试落点`)
120
+ }
121
+ assert.equal(after, before, `点击不得夺取前台(${before} → ${after})`)
122
+ // ⚠️这两条是命门:上面三条在"点击压根没生效"时照样全绿 —— "不抢前台"可以靠什么都不做换来。
123
+ assert.ok(five.display.endsWith('5'), `点了 5,读数应以 5 结尾,实得 ${five.display}`)
124
+ assert.ok(seven.display.endsWith('7'), `点了 7,读数应以 7 结尾,实得 ${seven.display}`)
125
+ console.log('✅ 后台盲点:走 AX、不抢前台、且两次都真的点中了')
126
+ } finally {
127
+ await killCalculator(ask).catch(() => {})
128
+ }