@moxxy/plugin-computer-control 0.41.2 → 0.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (269) hide show
  1. package/bin/win32-x64/moxxy-computer.exe +0 -0
  2. package/bin/win32-x64/moxxy-computer.exe.json +1 -1
  3. package/dist/backend/access.d.ts +129 -0
  4. package/dist/backend/access.d.ts.map +1 -0
  5. package/dist/backend/access.js +158 -0
  6. package/dist/backend/access.js.map +1 -0
  7. package/dist/backend/app-hints.d.ts +14 -0
  8. package/dist/backend/app-hints.d.ts.map +1 -0
  9. package/dist/backend/app-hints.js +30 -0
  10. package/dist/backend/app-hints.js.map +1 -0
  11. package/dist/backend/backend.d.ts +70 -0
  12. package/dist/backend/backend.d.ts.map +1 -0
  13. package/dist/backend/backend.js +466 -0
  14. package/dist/backend/backend.js.map +1 -0
  15. package/dist/backend/rpc.d.ts +1438 -0
  16. package/dist/backend/rpc.d.ts.map +1 -0
  17. package/dist/backend/rpc.js +75 -0
  18. package/dist/backend/rpc.js.map +1 -0
  19. package/dist/backend/turn-controls.d.ts +20 -0
  20. package/dist/backend/turn-controls.d.ts.map +1 -0
  21. package/dist/backend/turn-controls.js +115 -0
  22. package/dist/backend/turn-controls.js.map +1 -0
  23. package/dist/contract/guidance.d.ts +12 -0
  24. package/dist/contract/guidance.d.ts.map +1 -0
  25. package/dist/contract/guidance.js +42 -0
  26. package/dist/contract/guidance.js.map +1 -0
  27. package/dist/contract/image.d.ts +29 -0
  28. package/dist/contract/image.d.ts.map +1 -0
  29. package/dist/contract/image.js +46 -0
  30. package/dist/contract/image.js.map +1 -0
  31. package/dist/contract/keys.d.ts +15 -0
  32. package/dist/contract/keys.d.ts.map +1 -0
  33. package/dist/contract/keys.js +118 -0
  34. package/dist/contract/keys.js.map +1 -0
  35. package/dist/contract/outcome.d.ts +61 -0
  36. package/dist/contract/outcome.d.ts.map +1 -0
  37. package/dist/contract/outcome.js +63 -0
  38. package/dist/contract/outcome.js.map +1 -0
  39. package/dist/contract/progress.d.ts +27 -0
  40. package/dist/contract/progress.d.ts.map +1 -0
  41. package/dist/contract/progress.js +36 -0
  42. package/dist/contract/progress.js.map +1 -0
  43. package/dist/contract/tools.d.ts +338 -0
  44. package/dist/contract/tools.d.ts.map +1 -0
  45. package/dist/contract/tools.js +195 -0
  46. package/dist/contract/tools.js.map +1 -0
  47. package/dist/contract/untrusted.d.ts +3 -0
  48. package/dist/contract/untrusted.d.ts.map +1 -0
  49. package/dist/contract/untrusted.js +8 -0
  50. package/dist/contract/untrusted.js.map +1 -0
  51. package/dist/helper/artifact.d.ts +41 -0
  52. package/dist/helper/artifact.d.ts.map +1 -0
  53. package/dist/helper/artifact.js +189 -0
  54. package/dist/helper/artifact.js.map +1 -0
  55. package/dist/helper/protocol.d.ts +80 -0
  56. package/dist/helper/protocol.d.ts.map +1 -0
  57. package/dist/helper/protocol.js +49 -0
  58. package/dist/helper/protocol.js.map +1 -0
  59. package/dist/helper/transport.d.ts +43 -0
  60. package/dist/helper/transport.d.ts.map +1 -0
  61. package/dist/{windows → helper}/transport.js +45 -19
  62. package/dist/helper/transport.js.map +1 -0
  63. package/dist/index.d.ts +9 -22
  64. package/dist/index.d.ts.map +1 -1
  65. package/dist/index.js +35 -42
  66. package/dist/index.js.map +1 -1
  67. package/dist/jev/ladder.d.ts +32 -0
  68. package/dist/jev/ladder.d.ts.map +1 -0
  69. package/dist/jev/ladder.js +66 -0
  70. package/dist/jev/ladder.js.map +1 -0
  71. package/dist/jev/memory.d.ts +11 -0
  72. package/dist/jev/memory.d.ts.map +1 -0
  73. package/dist/jev/memory.js +16 -0
  74. package/dist/jev/memory.js.map +1 -0
  75. package/dist/jev/run.d.ts +93 -0
  76. package/dist/jev/run.d.ts.map +1 -0
  77. package/dist/jev/run.js +403 -0
  78. package/dist/jev/run.js.map +1 -0
  79. package/dist/jev/train.d.ts +97 -0
  80. package/dist/jev/train.d.ts.map +1 -0
  81. package/dist/jev/train.js +28 -0
  82. package/dist/jev/train.js.map +1 -0
  83. package/dist/linux/profile.d.ts +6 -0
  84. package/dist/linux/profile.d.ts.map +1 -0
  85. package/dist/linux/profile.js +21 -0
  86. package/dist/linux/profile.js.map +1 -0
  87. package/dist/macos/profile.d.ts +5 -0
  88. package/dist/macos/profile.d.ts.map +1 -0
  89. package/dist/macos/profile.js +17 -0
  90. package/dist/macos/profile.js.map +1 -0
  91. package/dist/preview/controller.d.ts +127 -0
  92. package/dist/preview/controller.d.ts.map +1 -0
  93. package/dist/preview/controller.js +203 -0
  94. package/dist/preview/controller.js.map +1 -0
  95. package/dist/preview/surface.d.ts +9 -0
  96. package/dist/preview/surface.d.ts.map +1 -0
  97. package/dist/preview/surface.js +50 -0
  98. package/dist/preview/surface.js.map +1 -0
  99. package/dist/windows/maintenance.d.ts +1 -1
  100. package/dist/windows/maintenance.d.ts.map +1 -1
  101. package/dist/windows/maintenance.js +6 -5
  102. package/dist/windows/maintenance.js.map +1 -1
  103. package/dist/windows/profile.d.ts +5 -0
  104. package/dist/windows/profile.d.ts.map +1 -0
  105. package/dist/windows/profile.js +17 -0
  106. package/dist/windows/profile.js.map +1 -0
  107. package/learned/README.md +26 -0
  108. package/learned/cases/com.apple.calculator.json +395 -0
  109. package/learned/cases/com.apple.finder.json +115 -0
  110. package/learned/cases/com.apple.safari.json +112 -0
  111. package/learned/cases/com.apple.systempreferences.json +255 -0
  112. package/learned/com.apple.calculator-eace95fc.json +334 -0
  113. package/learned/com.apple.finder-27cf6ce8.json +260 -0
  114. package/learned/com.apple.safari-7cd9df4f.json +143 -0
  115. package/learned/com.apple.systempreferences-02cf0b8b.json +535 -0
  116. package/package.json +13 -7
  117. package/scripts/promote-learned.mjs +8 -0
  118. package/scripts/summarize-trial.mjs +68 -0
  119. package/scripts/train-learned.mjs +64 -0
  120. package/skills/computer-apps/blender.md +29 -0
  121. package/skills/computer-apps/browsers.md +50 -0
  122. package/skills/computer-apps/design-tools.md +37 -0
  123. package/skills/computer-apps/finder.md +23 -0
  124. package/skills/computer-apps/office.md +46 -0
  125. package/skills/computer-apps/video-editors.md +44 -0
  126. package/skills/computer-control.md +154 -191
  127. package/src/backend/access.test.ts +158 -0
  128. package/src/backend/access.ts +168 -0
  129. package/src/backend/app-hints.test.ts +59 -0
  130. package/src/backend/app-hints.ts +39 -0
  131. package/src/backend/backend.test.ts +725 -0
  132. package/src/backend/backend.ts +482 -0
  133. package/src/backend/contract-helper.fixture.mjs +138 -0
  134. package/src/backend/helper.fixture.ts +38 -0
  135. package/src/backend/rpc.ts +90 -0
  136. package/src/backend/turn-controls.test.ts +178 -0
  137. package/src/backend/turn-controls.ts +120 -0
  138. package/src/contract/guidance.test.ts +75 -0
  139. package/src/contract/guidance.ts +48 -0
  140. package/src/contract/image.test.ts +74 -0
  141. package/src/contract/image.ts +54 -0
  142. package/src/contract/keys.test.ts +103 -0
  143. package/src/contract/keys.ts +117 -0
  144. package/src/contract/outcome.test.ts +71 -0
  145. package/src/contract/outcome.ts +72 -0
  146. package/src/contract/progress.test.ts +56 -0
  147. package/src/contract/progress.ts +43 -0
  148. package/src/contract/tools.test.ts +210 -0
  149. package/src/contract/tools.ts +212 -0
  150. package/src/contract/untrusted.test.ts +20 -0
  151. package/src/contract/untrusted.ts +8 -0
  152. package/src/helper/artifact.test.ts +131 -0
  153. package/src/helper/artifact.ts +182 -0
  154. package/src/helper/protocol.test.ts +29 -0
  155. package/src/helper/protocol.ts +50 -0
  156. package/src/{windows → helper}/transport.test.ts +76 -19
  157. package/src/{windows → helper}/transport.ts +54 -16
  158. package/src/index.test.ts +112 -0
  159. package/src/index.ts +36 -53
  160. package/src/jev/ladder.test.ts +113 -0
  161. package/src/jev/ladder.ts +88 -0
  162. package/src/jev/memory.ts +20 -0
  163. package/src/jev/run.test.ts +700 -0
  164. package/src/jev/run.ts +443 -0
  165. package/src/jev/train.test.ts +40 -0
  166. package/src/jev/train.ts +41 -0
  167. package/src/linux/helper.test.ts +481 -0
  168. package/src/linux/profile.ts +25 -0
  169. package/src/macos/helper.test.ts +988 -0
  170. package/src/macos/profile.ts +19 -0
  171. package/src/preview/controller.test.ts +308 -0
  172. package/src/preview/controller.ts +262 -0
  173. package/src/preview/surface.test.ts +68 -0
  174. package/src/preview/surface.ts +45 -0
  175. package/src/skill.test.ts +34 -0
  176. package/src/windows/maintenance.ts +8 -7
  177. package/src/windows/profile.ts +19 -0
  178. package/dist/shell.d.ts +0 -56
  179. package/dist/shell.d.ts.map +0 -1
  180. package/dist/shell.js +0 -189
  181. package/dist/shell.js.map +0 -1
  182. package/dist/temporary-files.d.ts +0 -2
  183. package/dist/temporary-files.d.ts.map +0 -1
  184. package/dist/temporary-files.js +0 -10
  185. package/dist/temporary-files.js.map +0 -1
  186. package/dist/tools/applescript.d.ts +0 -2
  187. package/dist/tools/applescript.d.ts.map +0 -1
  188. package/dist/tools/applescript.js +0 -50
  189. package/dist/tools/applescript.js.map +0 -1
  190. package/dist/tools/click.d.ts +0 -2
  191. package/dist/tools/click.d.ts.map +0 -1
  192. package/dist/tools/click.js +0 -58
  193. package/dist/tools/click.js.map +0 -1
  194. package/dist/tools/clipboard.d.ts +0 -2
  195. package/dist/tools/clipboard.d.ts.map +0 -1
  196. package/dist/tools/clipboard.js +0 -71
  197. package/dist/tools/clipboard.js.map +0 -1
  198. package/dist/tools/key.d.ts +0 -11
  199. package/dist/tools/key.d.ts.map +0 -1
  200. package/dist/tools/key.js +0 -143
  201. package/dist/tools/key.js.map +0 -1
  202. package/dist/tools/open.d.ts +0 -2
  203. package/dist/tools/open.d.ts.map +0 -1
  204. package/dist/tools/open.js +0 -91
  205. package/dist/tools/open.js.map +0 -1
  206. package/dist/tools/screenshot.d.ts +0 -2
  207. package/dist/tools/screenshot.d.ts.map +0 -1
  208. package/dist/tools/screenshot.js +0 -162
  209. package/dist/tools/screenshot.js.map +0 -1
  210. package/dist/tools/type.d.ts +0 -8
  211. package/dist/tools/type.d.ts.map +0 -1
  212. package/dist/tools/type.js +0 -65
  213. package/dist/tools/type.js.map +0 -1
  214. package/dist/windows/artifact.d.ts +0 -3
  215. package/dist/windows/artifact.d.ts.map +0 -1
  216. package/dist/windows/artifact.js +0 -57
  217. package/dist/windows/artifact.js.map +0 -1
  218. package/dist/windows/backend.d.ts +0 -11
  219. package/dist/windows/backend.d.ts.map +0 -1
  220. package/dist/windows/backend.js +0 -122
  221. package/dist/windows/backend.js.map +0 -1
  222. package/dist/windows/contracts.d.ts +0 -1627
  223. package/dist/windows/contracts.d.ts.map +0 -1
  224. package/dist/windows/contracts.js +0 -137
  225. package/dist/windows/contracts.js.map +0 -1
  226. package/dist/windows/control-service.d.ts +0 -12
  227. package/dist/windows/control-service.d.ts.map +0 -1
  228. package/dist/windows/control-service.js +0 -65
  229. package/dist/windows/control-service.js.map +0 -1
  230. package/dist/windows/guidance.d.ts +0 -3
  231. package/dist/windows/guidance.d.ts.map +0 -1
  232. package/dist/windows/guidance.js +0 -20
  233. package/dist/windows/guidance.js.map +0 -1
  234. package/dist/windows/protocol.d.ts +0 -9
  235. package/dist/windows/protocol.d.ts.map +0 -1
  236. package/dist/windows/protocol.js +0 -32
  237. package/dist/windows/protocol.js.map +0 -1
  238. package/dist/windows/transport.d.ts +0 -23
  239. package/dist/windows/transport.d.ts.map +0 -1
  240. package/dist/windows/transport.js.map +0 -1
  241. package/src/shell.test.ts +0 -186
  242. package/src/shell.ts +0 -213
  243. package/src/temporary-files.test.ts +0 -18
  244. package/src/temporary-files.ts +0 -9
  245. package/src/tools/applescript-serialize.test.ts +0 -74
  246. package/src/tools/applescript.ts +0 -53
  247. package/src/tools/click.ts +0 -60
  248. package/src/tools/clipboard.ts +0 -72
  249. package/src/tools/key.ts +0 -155
  250. package/src/tools/open.ts +0 -96
  251. package/src/tools/screenshot.test.ts +0 -137
  252. package/src/tools/screenshot.ts +0 -180
  253. package/src/tools/type.ts +0 -68
  254. package/src/tools.test.ts +0 -94
  255. package/src/windows/action-contracts.test.ts +0 -13
  256. package/src/windows/artifact.test.ts +0 -16
  257. package/src/windows/artifact.ts +0 -57
  258. package/src/windows/backend.test.ts +0 -41
  259. package/src/windows/backend.ts +0 -122
  260. package/src/windows/contracts.test.ts +0 -81
  261. package/src/windows/contracts.ts +0 -143
  262. package/src/windows/control-service.test.ts +0 -58
  263. package/src/windows/control-service.ts +0 -68
  264. package/src/windows/guidance.test.ts +0 -29
  265. package/src/windows/guidance.ts +0 -21
  266. package/src/windows/model-contract.test.ts +0 -37
  267. package/src/windows/protocol.ts +0 -27
  268. package/src/windows/text-contracts.test.ts +0 -14
  269. package/src/windows/window-typing.test.ts +0 -14
package/src/shell.ts DELETED
@@ -1,213 +0,0 @@
1
- import { spawn } from 'node:child_process';
2
- import { MoxxyError } from '@moxxy/sdk';
3
-
4
- export const IS_DARWIN = process.platform === 'darwin';
5
-
6
- /**
7
- * Hard ceiling on combined stdout+stderr a child may emit before it is
8
- * force-killed. The wall-clock timeout caps duration but not throughput —
9
- * a runaway or hostile child (e.g. `pbpaste` on a gigabyte clipboard, or an
10
- * `osascript` snippet that streams unbounded output) can accumulate hundreds
11
- * of MB well inside the timeout and OOM the host. 16 MB is far above any
12
- * legitimate tool output here (the screenshot path caps its own encoded file
13
- * separately) while still bounding worst-case memory.
14
- */
15
- export const MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
16
-
17
- /** Grace period after SIGTERM before escalating to SIGKILL. */
18
- const SIGKILL_GRACE_MS = 2_500;
19
-
20
- export interface ProcResult {
21
- readonly exitCode: number;
22
- readonly stdout: string;
23
- readonly stderr: string;
24
- /**
25
- * True when the child was force-killed because it exceeded `timeoutMs`.
26
- * A timed-out child still resolves (via 'close') with `exitCode: -1`, so
27
- * this flag is the only reliable way for callers to distinguish a timeout
28
- * from a genuine non-zero exit and surface an actionable message.
29
- */
30
- readonly timedOut: boolean;
31
- /** True when the child was force-killed because `opts.signal` aborted. */
32
- readonly aborted: boolean;
33
- /**
34
- * True when the child was force-killed because its combined stdout+stderr
35
- * exceeded `MAX_OUTPUT_BYTES`. The captured output is whatever had been
36
- * read up to the cap.
37
- */
38
- readonly tooLarge: boolean;
39
- }
40
-
41
- /**
42
- * Build a uniform failure suffix that names a timeout/abort when the process
43
- * was force-killed, so a stuck `osascript` (etc.) is reported as a clear
44
- * cause rather than a bare `exit -1`. Returns '' for a normal exit.
45
- */
46
- export function procFailureCause(proc: ProcResult, timeoutMs?: number): string {
47
- if (proc.tooLarge) return `output exceeded ${MAX_OUTPUT_BYTES} bytes (killed)`;
48
- if (proc.timedOut) {
49
- return timeoutMs ? `timed out after ${timeoutMs}ms` : 'timed out';
50
- }
51
- if (proc.aborted) return 'aborted (turn cancelled)';
52
- return '';
53
- }
54
-
55
- /**
56
- * Spawn a process with array-form args (no shell). Returns stdout +
57
- * stderr + exit code. Optional `input` is written to stdin. Optional
58
- * `signal` propagates aborts from the tool ctx so a stuck `osascript`
59
- * dies with the turn instead of hanging the parent.
60
- *
61
- * Never use this with string interpolation into a single command —
62
- * each argument MUST be a separate array entry. The `bash -c` shape
63
- * would re-introduce the shell-injection risk this helper exists to
64
- * eliminate.
65
- */
66
- export function runProcess(
67
- cmd: string,
68
- args: ReadonlyArray<string>,
69
- opts: {
70
- readonly input?: string | Buffer;
71
- readonly signal?: AbortSignal;
72
- readonly timeoutMs?: number;
73
- } = {},
74
- ): Promise<ProcResult> {
75
- return new Promise((resolve, reject) => {
76
- // An ALREADY-aborted signal never emits 'abort' to a listener added after
77
- // the fact (DOM semantics), so without this short-circuit a cancelled turn
78
- // would still spawn the child and run it to completion — defeating the
79
- // abort contract. Resolve immediately as an aborted no-op without spawning.
80
- if (opts.signal?.aborted) {
81
- resolve({
82
- exitCode: -1,
83
- stdout: '',
84
- stderr: '',
85
- timedOut: false,
86
- aborted: true,
87
- tooLarge: false,
88
- });
89
- return;
90
- }
91
- const child = spawn(cmd, [...args], { stdio: ['pipe', 'pipe', 'pipe'] });
92
- // Collect BOTH stdout and stderr as chunks and concat ONCE at close.
93
- // Re-concatenating the whole accumulated buffer on every 'data' event is
94
- // O(n^2) and churns the GC; decoding each chunk with `.toString('utf8')`
95
- // in isolation also corrupts any multibyte sequence (e.g. an osascript
96
- // error containing a non-ASCII app name) that straddles a chunk boundary.
97
- const stdoutChunks: Buffer[] = [];
98
- const stderrChunks: Buffer[] = [];
99
- let outputBytes = 0;
100
- let settled = false;
101
- let timedOut = false;
102
- let aborted = false;
103
- let tooLarge = false;
104
- let killTimer: ReturnType<typeof setTimeout> | null = null;
105
-
106
- // Force the child to die: SIGTERM first, then escalate to SIGKILL after a
107
- // grace period for a child that traps/ignores SIGTERM (a wedged GUI helper
108
- // or an osascript stuck behind an Accessibility prompt). Without the
109
- // escalation the tool's timeout/abort isn't actually enforced — the parent
110
- // hangs until the child closes on its own, which may be never.
111
- const forceKill = (): void => {
112
- try {
113
- child.kill('SIGTERM');
114
- } catch {
115
- /* ignore */
116
- }
117
- if (killTimer) return;
118
- killTimer = setTimeout(() => {
119
- if (settled) return;
120
- try {
121
- child.kill('SIGKILL');
122
- } catch {
123
- /* ignore */
124
- }
125
- }, SIGKILL_GRACE_MS);
126
- killTimer.unref?.();
127
- };
128
-
129
- const onAbort = (): void => {
130
- if (settled) return;
131
- aborted = true;
132
- forceKill();
133
- };
134
- opts.signal?.addEventListener('abort', onAbort, { once: true });
135
-
136
- const timer = opts.timeoutMs
137
- ? setTimeout(() => {
138
- if (settled) return;
139
- timedOut = true;
140
- forceKill();
141
- }, opts.timeoutMs)
142
- : null;
143
-
144
- child.stdout.on('data', (chunk: Buffer) => {
145
- outputBytes += chunk.length;
146
- // Once over the cap we stop retaining further chunks (the child is being
147
- // killed) so a flood between the cap-trip and SIGTERM landing can't keep
148
- // growing memory; we still account the bytes to keep `outputBytes` honest.
149
- if (!tooLarge) stdoutChunks.push(chunk);
150
- if (!tooLarge && outputBytes > MAX_OUTPUT_BYTES) {
151
- tooLarge = true;
152
- forceKill();
153
- }
154
- });
155
- child.stderr.on('data', (chunk: Buffer) => {
156
- outputBytes += chunk.length;
157
- if (!tooLarge) stderrChunks.push(chunk);
158
- if (!tooLarge && outputBytes > MAX_OUTPUT_BYTES) {
159
- tooLarge = true;
160
- forceKill();
161
- }
162
- });
163
- // A child that exits/closes stdin before our write completes makes the
164
- // stdin Writable emit 'error' (EPIPE/ECONNRESET). With no listener that
165
- // throws and surfaces as an unhandled rejection that can take down the
166
- // parent — swallow it; the child 'error'/'close' path reports the real
167
- // failure. Same defensiveness on stdout/stderr for symmetry.
168
- child.stdin.on('error', () => {});
169
- child.stdout.on('error', () => {});
170
- child.stderr.on('error', () => {});
171
- const cleanup = (): void => {
172
- if (timer) clearTimeout(timer);
173
- if (killTimer) clearTimeout(killTimer);
174
- opts.signal?.removeEventListener('abort', onAbort);
175
- };
176
- child.once('error', (err) => {
177
- if (settled) return;
178
- settled = true;
179
- cleanup();
180
- reject(err);
181
- });
182
- child.once('close', (code) => {
183
- if (settled) return;
184
- settled = true;
185
- cleanup();
186
- resolve({
187
- exitCode: code ?? -1,
188
- stdout: Buffer.concat(stdoutChunks).toString('utf8'),
189
- stderr: Buffer.concat(stderrChunks).toString('utf8'),
190
- timedOut,
191
- aborted,
192
- tooLarge,
193
- });
194
- });
195
-
196
- if (opts.input !== undefined) {
197
- child.stdin.end(opts.input);
198
- } else {
199
- child.stdin.end();
200
- }
201
- });
202
- }
203
-
204
- /** Throw a clear error when a tool is invoked on a non-darwin host. */
205
- export function ensureDarwin(toolName: string): void {
206
- if (!IS_DARWIN) {
207
- throw new MoxxyError({
208
- code: 'TOOL_ERROR',
209
- message: `${toolName}: @moxxy/plugin-computer-control currently only supports macOS (process.platform = ${process.platform})`,
210
- context: { tool: toolName, platform: process.platform },
211
- });
212
- }
213
- }
@@ -1,18 +0,0 @@
1
- import { mkdtemp, writeFile, access, rm } from 'node:fs/promises';
2
- import { tmpdir } from 'node:os';
3
- import { join } from 'node:path';
4
- import { expect, it } from 'vitest';
5
- import { withTemporaryFiles } from './temporary-files.js';
6
-
7
- it('cleans both screenshot files when conversion rejects after creating its output', async () => {
8
- const dir = await mkdtemp(join(tmpdir(), 'moxxy-capture-test-'));
9
- const files = [join(dir, 'capture.png'), join(dir, 'output.jpg')];
10
- try {
11
- const failure = new Error('converter aborted');
12
- await expect(withTemporaryFiles(files, async () => {
13
- for (const file of files) await writeFile(file, 'partial image');
14
- throw failure;
15
- })).rejects.toBe(failure);
16
- for (const file of files) await expect(access(file)).rejects.toThrow();
17
- } finally { await rm(dir, { recursive: true, force: true }); }
18
- });
@@ -1,9 +0,0 @@
1
- import { rm } from 'node:fs/promises';
2
-
3
- export async function withTemporaryFiles<T>(paths: readonly string[], work: () => Promise<T>): Promise<T> {
4
- try {
5
- return await work();
6
- } finally {
7
- await Promise.all(paths.map((path) => rm(path, { force: true })));
8
- }
9
- }
@@ -1,74 +0,0 @@
1
- import { MoxxyError } from '@moxxy/sdk';
2
- import { describe, expect, it } from 'vitest';
3
- import { buildKeyScript } from './key.js';
4
- import { toAppleScriptString } from './type.js';
5
-
6
- describe('toAppleScriptString', () => {
7
- it('wraps a plain string in quotes', () => {
8
- expect(toAppleScriptString('hello')).toBe('"hello"');
9
- });
10
-
11
- it('escapes embedded double-quotes and backslashes', () => {
12
- // `a"b` -> a\"b ; backslashes are doubled.
13
- expect(toAppleScriptString('a"b')).toBe('"a\\"b"');
14
- expect(toAppleScriptString('a\\b')).toBe('"a\\\\b"');
15
- // A backslash followed by a quote: both escaped, order preserved.
16
- expect(toAppleScriptString('\\"')).toBe('"\\\\\\""');
17
- });
18
-
19
- it('splits newlines into separate literals joined by AppleScript return', () => {
20
- expect(toAppleScriptString('a\nb')).toBe('"a" & return & "b"');
21
- expect(toAppleScriptString('one\ntwo\nthree')).toBe(
22
- '"one" & return & "two" & return & "three"',
23
- );
24
- });
25
- });
26
-
27
- describe('buildKeyScript', () => {
28
- it('uses key code for named keys with no modifiers', () => {
29
- expect(buildKeyScript('return', [])).toBe('tell application "System Events" to key code 36');
30
- expect(buildKeyScript('escape', [])).toBe('tell application "System Events" to key code 53');
31
- });
32
-
33
- it('is case-insensitive for named keys', () => {
34
- expect(buildKeyScript('Return', [])).toBe('tell application "System Events" to key code 36');
35
- });
36
-
37
- it('appends a using-clause mapping short modifier names to AppleScript', () => {
38
- expect(buildKeyScript('return', ['cmd'])).toBe(
39
- 'tell application "System Events" to key code 36 using {command down}',
40
- );
41
- expect(buildKeyScript('c', ['cmd', 'shift'])).toBe(
42
- 'tell application "System Events" to keystroke "c" using {command down, shift down}',
43
- );
44
- expect(buildKeyScript('a', ['option', 'control'])).toBe(
45
- 'tell application "System Events" to keystroke "a" using {option down, control down}',
46
- );
47
- });
48
-
49
- it('uses keystroke for single characters and escapes them', () => {
50
- expect(buildKeyScript('/', [])).toBe('tell application "System Events" to keystroke "/"');
51
- expect(buildKeyScript('"', [])).toBe('tell application "System Events" to keystroke "\\""');
52
- });
53
-
54
- it('routes a single literal whitespace char to its key code (chord-safe)', () => {
55
- // ctrl+space must press the Space key, not type a space char with control
56
- // held — so it goes through `key code 49`, not `keystroke " "`.
57
- expect(buildKeyScript(' ', ['control'])).toBe(
58
- 'tell application "System Events" to key code 49 using {control down}',
59
- );
60
- expect(buildKeyScript('\t', [])).toBe('tell application "System Events" to key code 48');
61
- expect(buildKeyScript('\r', [])).toBe('tell application "System Events" to key code 36');
62
- expect(buildKeyScript('\n', [])).toBe('tell application "System Events" to key code 36');
63
- });
64
-
65
- it('throws a TOOL_ERROR for an unknown multi-char key', () => {
66
- expect(() => buildKeyScript('notakey', [])).toThrow(MoxxyError);
67
- try {
68
- buildKeyScript('notakey', []);
69
- } catch (err) {
70
- expect((err as MoxxyError).code).toBe('TOOL_ERROR');
71
- expect((err as MoxxyError).message).toContain('unknown key "notakey"');
72
- }
73
- });
74
- });
@@ -1,53 +0,0 @@
1
- import { defineTool, MoxxyError, z } from '@moxxy/sdk';
2
- import { ensureDarwin, procFailureCause, runProcess } from '../shell.js';
3
-
4
- export const applescriptTool = defineTool({
5
- name: 'computer_applescript',
6
- description:
7
- 'Escape hatch: run an arbitrary AppleScript snippet via `osascript`. Use ' +
8
- 'for anything the dedicated tools can\'t cover (frontmost app name, list ' +
9
- 'open Safari tabs, drive a specific app via its scripting dictionary, etc.). ' +
10
- 'Returns the script\'s stdout. The user approves every invocation — keep ' +
11
- 'scripts focused and reversible.',
12
- inputSchema: z.object({
13
- script: z
14
- .string()
15
- .min(1)
16
- .max(8000)
17
- .describe(
18
- 'AppleScript source. Multi-line is fine — pass `\\n` between statements. ' +
19
- 'For JavaScript for Automation, prefix with `#!/usr/bin/osascript -l JavaScript`',
20
- ),
21
- }),
22
- permission: { action: 'prompt' },
23
- // Arbitrary AppleScript is an escape hatch on par with a shell: the script
24
- // can itself shell out (`do shell script`) and reach the network, so the
25
- // honest bounds are subprocess + any net — a confining isolator must not
26
- // assume osascript stays offline. Budget = the 30s child timeout + kill grace.
27
- isolation: {
28
- capabilities: {
29
- subprocess: true,
30
- commands: ['osascript'],
31
- net: { mode: 'any' },
32
- timeMs: 40_000,
33
- },
34
- },
35
- async handler({ script }, ctx) {
36
- ensureDarwin('computer_applescript');
37
- const proc = await runProcess('osascript', ['-e', script], {
38
- ...(ctx.signal ? { signal: ctx.signal } : {}),
39
- timeoutMs: 30_000,
40
- });
41
- if (proc.exitCode !== 0) {
42
- const cause = procFailureCause(proc, 30_000);
43
- throw new MoxxyError({
44
- code: 'TOOL_ERROR',
45
- message: cause
46
- ? `osascript ${cause}`
47
- : `osascript failed (exit ${proc.exitCode}): ${proc.stderr.trim() || '(no error message)'}`,
48
- context: { tool: 'computer_applescript', exitCode: proc.exitCode, timedOut: proc.timedOut ? 1 : 0 },
49
- });
50
- }
51
- return { ok: true, output: proc.stdout.trim() };
52
- },
53
- });
@@ -1,60 +0,0 @@
1
- import { defineTool, MoxxyError, z } from '@moxxy/sdk';
2
- import { ensureDarwin, procFailureCause, runProcess } from '../shell.js';
3
-
4
- export const clickTool = defineTool({
5
- name: 'computer_click',
6
- description:
7
- 'Click the mouse at screen-pixel coordinates (top-left origin). Uses macOS ' +
8
- 'System Events; requires Accessibility permission on first use. Pass count=2 ' +
9
- 'for a double-click. Right/middle buttons are not supported via this tool — ' +
10
- 'use computer_applescript for those (rare).',
11
- inputSchema: z.object({
12
- x: z.number().int().min(0).describe('X pixel from the top-left of the display.'),
13
- y: z.number().int().min(0).describe('Y pixel from the top-left of the display.'),
14
- count: z
15
- .number()
16
- .int()
17
- .min(1)
18
- .max(3)
19
- .optional()
20
- .describe('Number of consecutive clicks. 1 = single, 2 = double, 3 = triple. Default 1.'),
21
- }),
22
- permission: { action: 'prompt' },
23
- isolation: {
24
- capabilities: {
25
- subprocess: true,
26
- commands: ['osascript'],
27
- net: { mode: 'none' },
28
- timeMs: 15_000,
29
- },
30
- },
31
- async handler({ x, y, count }, ctx) {
32
- ensureDarwin('computer_click');
33
- const n = count ?? 1;
34
- // `click at {x, y}` repeats N times via a repeat block. Use a
35
- // bare AppleScript string with numeric interpolation — no
36
- // user-supplied text reaches the script, so injection isn't a
37
- // risk here.
38
- const script =
39
- `tell application "System Events"\n` +
40
- ` repeat ${n} times\n` +
41
- ` click at {${Math.round(x)}, ${Math.round(y)}}\n` +
42
- ` end repeat\n` +
43
- `end tell`;
44
- const proc = await runProcess('osascript', ['-e', script], {
45
- ...(ctx.signal ? { signal: ctx.signal } : {}),
46
- timeoutMs: 10_000,
47
- });
48
- if (proc.exitCode !== 0) {
49
- const cause = procFailureCause(proc, 10_000);
50
- throw new MoxxyError({
51
- code: 'TOOL_ERROR',
52
- message: cause
53
- ? `click ${cause}`
54
- : `click failed (exit ${proc.exitCode}): ${proc.stderr.trim() || '(check Accessibility permission in System Settings → Privacy & Security → Accessibility)'}`,
55
- context: { tool: 'computer_click', exitCode: proc.exitCode, timedOut: proc.timedOut ? 1 : 0 },
56
- });
57
- }
58
- return { ok: true, x, y, count: n };
59
- },
60
- });
@@ -1,72 +0,0 @@
1
- import { defineTool, MoxxyError, z } from '@moxxy/sdk';
2
- import { ensureDarwin, procFailureCause, runProcess } from '../shell.js';
3
-
4
- export const clipboardTool = defineTool({
5
- name: 'computer_clipboard',
6
- icon: 'clipboard',
7
- description:
8
- 'Read from or write to the macOS clipboard. Use `read` to fetch what the ' +
9
- 'user just copied; use `write` to stage text the user can then paste with ' +
10
- '⌘V. Writing the clipboard does NOT trigger a paste — call computer_key ' +
11
- 'with cmd+v after if you need to paste.',
12
- inputSchema: z.object({
13
- action: z.enum(['read', 'write']),
14
- text: z
15
- .string()
16
- .max(64_000)
17
- .optional()
18
- .describe('Text to put on the clipboard. Required when action="write".'),
19
- }),
20
- permission: { action: 'prompt' },
21
- isolation: {
22
- capabilities: {
23
- subprocess: true,
24
- commands: ['pbpaste', 'pbcopy'],
25
- net: { mode: 'none' },
26
- timeMs: 10_000,
27
- },
28
- },
29
- async handler({ action, text }, ctx) {
30
- ensureDarwin('computer_clipboard');
31
- if (action === 'read') {
32
- const proc = await runProcess('pbpaste', [], {
33
- ...(ctx.signal ? { signal: ctx.signal } : {}),
34
- timeoutMs: 5_000,
35
- });
36
- if (proc.exitCode !== 0) {
37
- const cause = procFailureCause(proc, 5_000);
38
- throw new MoxxyError({
39
- code: 'TOOL_ERROR',
40
- message: cause
41
- ? `pbpaste ${cause}`
42
- : `pbpaste failed (exit ${proc.exitCode}): ${proc.stderr.trim()}`,
43
- context: { tool: 'computer_clipboard', exitCode: proc.exitCode, timedOut: proc.timedOut ? 1 : 0 },
44
- });
45
- }
46
- return { ok: true, text: proc.stdout };
47
- }
48
- if (text === undefined) {
49
- throw new MoxxyError({
50
- code: 'TOOL_ERROR',
51
- message: 'computer_clipboard: `text` is required when action="write"',
52
- context: { tool: 'computer_clipboard' },
53
- });
54
- }
55
- const proc = await runProcess('pbcopy', [], {
56
- ...(ctx.signal ? { signal: ctx.signal } : {}),
57
- input: text,
58
- timeoutMs: 5_000,
59
- });
60
- if (proc.exitCode !== 0) {
61
- const cause = procFailureCause(proc, 5_000);
62
- throw new MoxxyError({
63
- code: 'TOOL_ERROR',
64
- message: cause
65
- ? `pbcopy ${cause}`
66
- : `pbcopy failed (exit ${proc.exitCode}): ${proc.stderr.trim()}`,
67
- context: { tool: 'computer_clipboard', exitCode: proc.exitCode, timedOut: proc.timedOut ? 1 : 0 },
68
- });
69
- }
70
- return { ok: true, length: text.length };
71
- },
72
- });
package/src/tools/key.ts DELETED
@@ -1,155 +0,0 @@
1
- import { defineTool, MoxxyError, z } from '@moxxy/sdk';
2
- import { ensureDarwin, procFailureCause, runProcess } from '../shell.js';
3
-
4
- const MODIFIER_NAMES = ['cmd', 'shift', 'option', 'control'] as const;
5
- type Modifier = (typeof MODIFIER_NAMES)[number];
6
-
7
- /**
8
- * Named keys → macOS key codes used by AppleScript's
9
- * `key code N` command. Subset focused on what an automation agent
10
- * actually needs (navigation, editing, function keys). Letters and
11
- * digits go through `keystroke "x" using ...` instead of `key code`.
12
- */
13
- const KEY_CODES: Record<string, number> = {
14
- return: 36,
15
- enter: 36,
16
- tab: 48,
17
- space: 49,
18
- escape: 53,
19
- esc: 53,
20
- delete: 51,
21
- backspace: 51,
22
- forward_delete: 117,
23
- left: 123,
24
- right: 124,
25
- down: 125,
26
- up: 126,
27
- home: 115,
28
- end: 119,
29
- page_up: 116,
30
- page_down: 121,
31
- f1: 122,
32
- f2: 120,
33
- f3: 99,
34
- f4: 118,
35
- f5: 96,
36
- f6: 97,
37
- f7: 98,
38
- f8: 100,
39
- f9: 101,
40
- f10: 109,
41
- f11: 103,
42
- f12: 111,
43
- };
44
-
45
- /**
46
- * Single literal whitespace chars → their key codes. A bare ' '/'\t'/'\r'/'\n'
47
- * would otherwise be sent via `keystroke`, which types the character (wrong for
48
- * modifier chords like ctrl+space). Mirrors the matching KEY_CODES entries.
49
- */
50
- const WHITESPACE_KEY_CODES: Record<string, number> = {
51
- ' ': 49, // space
52
- '\t': 48, // tab
53
- '\r': 36, // return
54
- '\n': 36, // return
55
- };
56
-
57
- export const keyTool = defineTool({
58
- name: 'computer_key',
59
- description:
60
- 'Send a single key chord with optional modifiers. Use this for shortcuts ' +
61
- '(cmd+c, cmd+tab, cmd+shift+4) and named keys (return, tab, escape, arrows, ' +
62
- 'page_up/down, f1–f12). For typing arbitrary text, use computer_type.',
63
- inputSchema: z.object({
64
- key: z
65
- .string()
66
- .min(1)
67
- .describe(
68
- 'A single character ("a", "1", "/") OR a named key from the catalog: ' +
69
- Object.keys(KEY_CODES).join(', ') +
70
- '.',
71
- ),
72
- modifiers: z
73
- .array(z.enum(MODIFIER_NAMES))
74
- .optional()
75
- .describe(
76
- 'Held modifiers. Common combos: ["cmd"] for cmd+key, ["cmd","shift"], ["control","option"].',
77
- ),
78
- }),
79
- permission: { action: 'prompt' },
80
- isolation: {
81
- capabilities: {
82
- subprocess: true,
83
- commands: ['osascript'],
84
- net: { mode: 'none' },
85
- timeMs: 15_000,
86
- },
87
- },
88
- async handler({ key, modifiers }, ctx) {
89
- ensureDarwin('computer_key');
90
- const mods = modifiers ?? [];
91
- const script = buildKeyScript(key, mods);
92
- const proc = await runProcess('osascript', ['-e', script], {
93
- ...(ctx.signal ? { signal: ctx.signal } : {}),
94
- timeoutMs: 10_000,
95
- });
96
- if (proc.exitCode !== 0) {
97
- const cause = procFailureCause(proc, 10_000);
98
- throw new MoxxyError({
99
- code: 'TOOL_ERROR',
100
- message: cause
101
- ? `key ${cause}`
102
- : `key failed (exit ${proc.exitCode}): ${proc.stderr.trim() || '(check Accessibility permission)'}`,
103
- context: { tool: 'computer_key', exitCode: proc.exitCode, timedOut: proc.timedOut ? 1 : 0 },
104
- });
105
- }
106
- return { ok: true, key, modifiers: mods };
107
- },
108
- });
109
-
110
- /**
111
- * Build the `osascript -e` body for a single key chord. Named keys go
112
- * through `key code N`, single characters through `keystroke "x"`, and a
113
- * multi-char unknown key throws. Pure (no I/O) so it can be unit-tested.
114
- */
115
- export function buildKeyScript(key: string, modifiers: ReadonlyArray<Modifier>): string {
116
- const usingClause =
117
- modifiers.length > 0 ? ` using {${modifiers.map(modifierClause).join(', ')}}` : '';
118
- const lower = key.toLowerCase();
119
- if (lower in KEY_CODES) {
120
- return `tell application "System Events" to key code ${KEY_CODES[lower]}${usingClause}`;
121
- }
122
- // A single literal whitespace char (' ', '\t', '\r', '\n') keystroked with a
123
- // held modifier types the character rather than pressing the key, breaking
124
- // chords like ctrl+space. Route it to the corresponding key code instead.
125
- if (key.length === 1) {
126
- const wsCode = WHITESPACE_KEY_CODES[key];
127
- if (wsCode !== undefined) {
128
- return `tell application "System Events" to key code ${wsCode}${usingClause}`;
129
- }
130
- }
131
- if (key.length === 1) {
132
- const literal = `"${key.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
133
- return `tell application "System Events" to keystroke ${literal}${usingClause}`;
134
- }
135
- throw new MoxxyError({
136
- code: 'TOOL_ERROR',
137
- message: `unknown key "${key}". Use a single character or one of: ${Object.keys(KEY_CODES).join(', ')}.`,
138
- context: { tool: 'computer_key', key },
139
- });
140
- }
141
-
142
- function modifierClause(m: Modifier): string {
143
- // AppleScript uses "command down", "shift down", "option down",
144
- // "control down" — map our short names.
145
- switch (m) {
146
- case 'cmd':
147
- return 'command down';
148
- case 'shift':
149
- return 'shift down';
150
- case 'option':
151
- return 'option down';
152
- case 'control':
153
- return 'control down';
154
- }
155
- }