@moxxy/plugin-computer-control 0.41.2 → 0.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (269) hide show
  1. package/bin/win32-x64/moxxy-computer.exe +0 -0
  2. package/bin/win32-x64/moxxy-computer.exe.json +1 -1
  3. package/dist/backend/access.d.ts +129 -0
  4. package/dist/backend/access.d.ts.map +1 -0
  5. package/dist/backend/access.js +158 -0
  6. package/dist/backend/access.js.map +1 -0
  7. package/dist/backend/app-hints.d.ts +14 -0
  8. package/dist/backend/app-hints.d.ts.map +1 -0
  9. package/dist/backend/app-hints.js +30 -0
  10. package/dist/backend/app-hints.js.map +1 -0
  11. package/dist/backend/backend.d.ts +70 -0
  12. package/dist/backend/backend.d.ts.map +1 -0
  13. package/dist/backend/backend.js +466 -0
  14. package/dist/backend/backend.js.map +1 -0
  15. package/dist/backend/rpc.d.ts +1438 -0
  16. package/dist/backend/rpc.d.ts.map +1 -0
  17. package/dist/backend/rpc.js +75 -0
  18. package/dist/backend/rpc.js.map +1 -0
  19. package/dist/backend/turn-controls.d.ts +20 -0
  20. package/dist/backend/turn-controls.d.ts.map +1 -0
  21. package/dist/backend/turn-controls.js +115 -0
  22. package/dist/backend/turn-controls.js.map +1 -0
  23. package/dist/contract/guidance.d.ts +12 -0
  24. package/dist/contract/guidance.d.ts.map +1 -0
  25. package/dist/contract/guidance.js +42 -0
  26. package/dist/contract/guidance.js.map +1 -0
  27. package/dist/contract/image.d.ts +29 -0
  28. package/dist/contract/image.d.ts.map +1 -0
  29. package/dist/contract/image.js +46 -0
  30. package/dist/contract/image.js.map +1 -0
  31. package/dist/contract/keys.d.ts +15 -0
  32. package/dist/contract/keys.d.ts.map +1 -0
  33. package/dist/contract/keys.js +118 -0
  34. package/dist/contract/keys.js.map +1 -0
  35. package/dist/contract/outcome.d.ts +61 -0
  36. package/dist/contract/outcome.d.ts.map +1 -0
  37. package/dist/contract/outcome.js +63 -0
  38. package/dist/contract/outcome.js.map +1 -0
  39. package/dist/contract/progress.d.ts +27 -0
  40. package/dist/contract/progress.d.ts.map +1 -0
  41. package/dist/contract/progress.js +36 -0
  42. package/dist/contract/progress.js.map +1 -0
  43. package/dist/contract/tools.d.ts +338 -0
  44. package/dist/contract/tools.d.ts.map +1 -0
  45. package/dist/contract/tools.js +195 -0
  46. package/dist/contract/tools.js.map +1 -0
  47. package/dist/contract/untrusted.d.ts +3 -0
  48. package/dist/contract/untrusted.d.ts.map +1 -0
  49. package/dist/contract/untrusted.js +8 -0
  50. package/dist/contract/untrusted.js.map +1 -0
  51. package/dist/helper/artifact.d.ts +41 -0
  52. package/dist/helper/artifact.d.ts.map +1 -0
  53. package/dist/helper/artifact.js +189 -0
  54. package/dist/helper/artifact.js.map +1 -0
  55. package/dist/helper/protocol.d.ts +80 -0
  56. package/dist/helper/protocol.d.ts.map +1 -0
  57. package/dist/helper/protocol.js +49 -0
  58. package/dist/helper/protocol.js.map +1 -0
  59. package/dist/helper/transport.d.ts +43 -0
  60. package/dist/helper/transport.d.ts.map +1 -0
  61. package/dist/{windows → helper}/transport.js +45 -19
  62. package/dist/helper/transport.js.map +1 -0
  63. package/dist/index.d.ts +9 -22
  64. package/dist/index.d.ts.map +1 -1
  65. package/dist/index.js +35 -42
  66. package/dist/index.js.map +1 -1
  67. package/dist/jev/ladder.d.ts +32 -0
  68. package/dist/jev/ladder.d.ts.map +1 -0
  69. package/dist/jev/ladder.js +66 -0
  70. package/dist/jev/ladder.js.map +1 -0
  71. package/dist/jev/memory.d.ts +11 -0
  72. package/dist/jev/memory.d.ts.map +1 -0
  73. package/dist/jev/memory.js +16 -0
  74. package/dist/jev/memory.js.map +1 -0
  75. package/dist/jev/run.d.ts +93 -0
  76. package/dist/jev/run.d.ts.map +1 -0
  77. package/dist/jev/run.js +403 -0
  78. package/dist/jev/run.js.map +1 -0
  79. package/dist/jev/train.d.ts +97 -0
  80. package/dist/jev/train.d.ts.map +1 -0
  81. package/dist/jev/train.js +28 -0
  82. package/dist/jev/train.js.map +1 -0
  83. package/dist/linux/profile.d.ts +6 -0
  84. package/dist/linux/profile.d.ts.map +1 -0
  85. package/dist/linux/profile.js +21 -0
  86. package/dist/linux/profile.js.map +1 -0
  87. package/dist/macos/profile.d.ts +5 -0
  88. package/dist/macos/profile.d.ts.map +1 -0
  89. package/dist/macos/profile.js +17 -0
  90. package/dist/macos/profile.js.map +1 -0
  91. package/dist/preview/controller.d.ts +127 -0
  92. package/dist/preview/controller.d.ts.map +1 -0
  93. package/dist/preview/controller.js +203 -0
  94. package/dist/preview/controller.js.map +1 -0
  95. package/dist/preview/surface.d.ts +9 -0
  96. package/dist/preview/surface.d.ts.map +1 -0
  97. package/dist/preview/surface.js +50 -0
  98. package/dist/preview/surface.js.map +1 -0
  99. package/dist/windows/maintenance.d.ts +1 -1
  100. package/dist/windows/maintenance.d.ts.map +1 -1
  101. package/dist/windows/maintenance.js +6 -5
  102. package/dist/windows/maintenance.js.map +1 -1
  103. package/dist/windows/profile.d.ts +5 -0
  104. package/dist/windows/profile.d.ts.map +1 -0
  105. package/dist/windows/profile.js +17 -0
  106. package/dist/windows/profile.js.map +1 -0
  107. package/learned/README.md +26 -0
  108. package/learned/cases/com.apple.calculator.json +395 -0
  109. package/learned/cases/com.apple.finder.json +115 -0
  110. package/learned/cases/com.apple.safari.json +112 -0
  111. package/learned/cases/com.apple.systempreferences.json +255 -0
  112. package/learned/com.apple.calculator-eace95fc.json +334 -0
  113. package/learned/com.apple.finder-27cf6ce8.json +260 -0
  114. package/learned/com.apple.safari-7cd9df4f.json +143 -0
  115. package/learned/com.apple.systempreferences-02cf0b8b.json +535 -0
  116. package/package.json +13 -7
  117. package/scripts/promote-learned.mjs +8 -0
  118. package/scripts/summarize-trial.mjs +68 -0
  119. package/scripts/train-learned.mjs +64 -0
  120. package/skills/computer-apps/blender.md +29 -0
  121. package/skills/computer-apps/browsers.md +50 -0
  122. package/skills/computer-apps/design-tools.md +37 -0
  123. package/skills/computer-apps/finder.md +23 -0
  124. package/skills/computer-apps/office.md +46 -0
  125. package/skills/computer-apps/video-editors.md +44 -0
  126. package/skills/computer-control.md +154 -191
  127. package/src/backend/access.test.ts +158 -0
  128. package/src/backend/access.ts +168 -0
  129. package/src/backend/app-hints.test.ts +59 -0
  130. package/src/backend/app-hints.ts +39 -0
  131. package/src/backend/backend.test.ts +725 -0
  132. package/src/backend/backend.ts +482 -0
  133. package/src/backend/contract-helper.fixture.mjs +138 -0
  134. package/src/backend/helper.fixture.ts +38 -0
  135. package/src/backend/rpc.ts +90 -0
  136. package/src/backend/turn-controls.test.ts +178 -0
  137. package/src/backend/turn-controls.ts +120 -0
  138. package/src/contract/guidance.test.ts +75 -0
  139. package/src/contract/guidance.ts +48 -0
  140. package/src/contract/image.test.ts +74 -0
  141. package/src/contract/image.ts +54 -0
  142. package/src/contract/keys.test.ts +103 -0
  143. package/src/contract/keys.ts +117 -0
  144. package/src/contract/outcome.test.ts +71 -0
  145. package/src/contract/outcome.ts +72 -0
  146. package/src/contract/progress.test.ts +56 -0
  147. package/src/contract/progress.ts +43 -0
  148. package/src/contract/tools.test.ts +210 -0
  149. package/src/contract/tools.ts +212 -0
  150. package/src/contract/untrusted.test.ts +20 -0
  151. package/src/contract/untrusted.ts +8 -0
  152. package/src/helper/artifact.test.ts +131 -0
  153. package/src/helper/artifact.ts +182 -0
  154. package/src/helper/protocol.test.ts +29 -0
  155. package/src/helper/protocol.ts +50 -0
  156. package/src/{windows → helper}/transport.test.ts +76 -19
  157. package/src/{windows → helper}/transport.ts +54 -16
  158. package/src/index.test.ts +112 -0
  159. package/src/index.ts +36 -53
  160. package/src/jev/ladder.test.ts +113 -0
  161. package/src/jev/ladder.ts +88 -0
  162. package/src/jev/memory.ts +20 -0
  163. package/src/jev/run.test.ts +700 -0
  164. package/src/jev/run.ts +443 -0
  165. package/src/jev/train.test.ts +40 -0
  166. package/src/jev/train.ts +41 -0
  167. package/src/linux/helper.test.ts +481 -0
  168. package/src/linux/profile.ts +25 -0
  169. package/src/macos/helper.test.ts +988 -0
  170. package/src/macos/profile.ts +19 -0
  171. package/src/preview/controller.test.ts +308 -0
  172. package/src/preview/controller.ts +262 -0
  173. package/src/preview/surface.test.ts +68 -0
  174. package/src/preview/surface.ts +45 -0
  175. package/src/skill.test.ts +34 -0
  176. package/src/windows/maintenance.ts +8 -7
  177. package/src/windows/profile.ts +19 -0
  178. package/dist/shell.d.ts +0 -56
  179. package/dist/shell.d.ts.map +0 -1
  180. package/dist/shell.js +0 -189
  181. package/dist/shell.js.map +0 -1
  182. package/dist/temporary-files.d.ts +0 -2
  183. package/dist/temporary-files.d.ts.map +0 -1
  184. package/dist/temporary-files.js +0 -10
  185. package/dist/temporary-files.js.map +0 -1
  186. package/dist/tools/applescript.d.ts +0 -2
  187. package/dist/tools/applescript.d.ts.map +0 -1
  188. package/dist/tools/applescript.js +0 -50
  189. package/dist/tools/applescript.js.map +0 -1
  190. package/dist/tools/click.d.ts +0 -2
  191. package/dist/tools/click.d.ts.map +0 -1
  192. package/dist/tools/click.js +0 -58
  193. package/dist/tools/click.js.map +0 -1
  194. package/dist/tools/clipboard.d.ts +0 -2
  195. package/dist/tools/clipboard.d.ts.map +0 -1
  196. package/dist/tools/clipboard.js +0 -71
  197. package/dist/tools/clipboard.js.map +0 -1
  198. package/dist/tools/key.d.ts +0 -11
  199. package/dist/tools/key.d.ts.map +0 -1
  200. package/dist/tools/key.js +0 -143
  201. package/dist/tools/key.js.map +0 -1
  202. package/dist/tools/open.d.ts +0 -2
  203. package/dist/tools/open.d.ts.map +0 -1
  204. package/dist/tools/open.js +0 -91
  205. package/dist/tools/open.js.map +0 -1
  206. package/dist/tools/screenshot.d.ts +0 -2
  207. package/dist/tools/screenshot.d.ts.map +0 -1
  208. package/dist/tools/screenshot.js +0 -162
  209. package/dist/tools/screenshot.js.map +0 -1
  210. package/dist/tools/type.d.ts +0 -8
  211. package/dist/tools/type.d.ts.map +0 -1
  212. package/dist/tools/type.js +0 -65
  213. package/dist/tools/type.js.map +0 -1
  214. package/dist/windows/artifact.d.ts +0 -3
  215. package/dist/windows/artifact.d.ts.map +0 -1
  216. package/dist/windows/artifact.js +0 -57
  217. package/dist/windows/artifact.js.map +0 -1
  218. package/dist/windows/backend.d.ts +0 -11
  219. package/dist/windows/backend.d.ts.map +0 -1
  220. package/dist/windows/backend.js +0 -122
  221. package/dist/windows/backend.js.map +0 -1
  222. package/dist/windows/contracts.d.ts +0 -1627
  223. package/dist/windows/contracts.d.ts.map +0 -1
  224. package/dist/windows/contracts.js +0 -137
  225. package/dist/windows/contracts.js.map +0 -1
  226. package/dist/windows/control-service.d.ts +0 -12
  227. package/dist/windows/control-service.d.ts.map +0 -1
  228. package/dist/windows/control-service.js +0 -65
  229. package/dist/windows/control-service.js.map +0 -1
  230. package/dist/windows/guidance.d.ts +0 -3
  231. package/dist/windows/guidance.d.ts.map +0 -1
  232. package/dist/windows/guidance.js +0 -20
  233. package/dist/windows/guidance.js.map +0 -1
  234. package/dist/windows/protocol.d.ts +0 -9
  235. package/dist/windows/protocol.d.ts.map +0 -1
  236. package/dist/windows/protocol.js +0 -32
  237. package/dist/windows/protocol.js.map +0 -1
  238. package/dist/windows/transport.d.ts +0 -23
  239. package/dist/windows/transport.d.ts.map +0 -1
  240. package/dist/windows/transport.js.map +0 -1
  241. package/src/shell.test.ts +0 -186
  242. package/src/shell.ts +0 -213
  243. package/src/temporary-files.test.ts +0 -18
  244. package/src/temporary-files.ts +0 -9
  245. package/src/tools/applescript-serialize.test.ts +0 -74
  246. package/src/tools/applescript.ts +0 -53
  247. package/src/tools/click.ts +0 -60
  248. package/src/tools/clipboard.ts +0 -72
  249. package/src/tools/key.ts +0 -155
  250. package/src/tools/open.ts +0 -96
  251. package/src/tools/screenshot.test.ts +0 -137
  252. package/src/tools/screenshot.ts +0 -180
  253. package/src/tools/type.ts +0 -68
  254. package/src/tools.test.ts +0 -94
  255. package/src/windows/action-contracts.test.ts +0 -13
  256. package/src/windows/artifact.test.ts +0 -16
  257. package/src/windows/artifact.ts +0 -57
  258. package/src/windows/backend.test.ts +0 -41
  259. package/src/windows/backend.ts +0 -122
  260. package/src/windows/contracts.test.ts +0 -81
  261. package/src/windows/contracts.ts +0 -143
  262. package/src/windows/control-service.test.ts +0 -58
  263. package/src/windows/control-service.ts +0 -68
  264. package/src/windows/guidance.test.ts +0 -29
  265. package/src/windows/guidance.ts +0 -21
  266. package/src/windows/model-contract.test.ts +0 -37
  267. package/src/windows/protocol.ts +0 -27
  268. package/src/windows/text-contracts.test.ts +0 -14
  269. package/src/windows/window-typing.test.ts +0 -14
package/src/tools/open.ts DELETED
@@ -1,96 +0,0 @@
1
- import { defineTool, MoxxyError, z } from '@moxxy/sdk';
2
- import { ensureDarwin, procFailureCause, runProcess } from '../shell.js';
3
-
4
- export const openTool = defineTool({
5
- name: 'computer_open',
6
- icon: 'globe',
7
- description:
8
- 'Open a URL, file path, or .app bundle via macOS `open`. Use this to ' +
9
- 'launch / activate a specific app or jump to a web page. The model should ' +
10
- 'prefer this over typing into Spotlight when the target is known.',
11
- inputSchema: z.object({
12
- target: z
13
- .string()
14
- .min(1)
15
- .describe(
16
- 'URL (https://...), file path (/Users/...), or app name (Safari). ' +
17
- 'For app names, prefer the `app` field — `target` is treated as a path.',
18
- )
19
- .optional(),
20
- app: z
21
- .string()
22
- .min(1)
23
- .max(120)
24
- .describe(
25
- 'Application name to activate (e.g. "Safari", "Visual Studio Code"). ' +
26
- 'When `target` is also set, the app opens `target` (e.g. open a file in VS Code).',
27
- )
28
- .optional(),
29
- }),
30
- permission: { action: 'prompt' },
31
- // `open` only messages LaunchServices: the file read / URL navigation happens
32
- // in the launched app, outside this process tree — hence net 'none'. The
33
- // broad fs.read is honest for this tool: any user-approved path is a
34
- // legitimate `target`, and the cap check validates path-shaped inputs
35
- // against these globs.
36
- isolation: {
37
- capabilities: {
38
- subprocess: true,
39
- commands: ['open'],
40
- fs: { read: ['/**'] },
41
- net: { mode: 'none' },
42
- timeMs: 15_000,
43
- },
44
- },
45
- async handler({ target, app }, ctx) {
46
- ensureDarwin('computer_open');
47
- if (!target && !app) {
48
- throw new MoxxyError({
49
- code: 'TOOL_ERROR',
50
- message: 'computer_open: at least one of `target` or `app` is required',
51
- context: { tool: 'computer_open' },
52
- });
53
- }
54
- // A target/app beginning with '-' (e.g. '-g', '-n', '-W') is parsed by
55
- // /usr/bin/open as an OPTION FLAG, not a path/URL/app — silently changing
56
- // behavior (open in background, new instance) the user did not approve.
57
- // Reject it: no legitimate path/URL/app name starts with a dash, and
58
- // spawn is array-form so this is the only argument-injection vector.
59
- for (const [field, value] of [
60
- ['target', target],
61
- ['app', app],
62
- ] as const) {
63
- if (value !== undefined && value.startsWith('-')) {
64
- throw new MoxxyError({
65
- code: 'TOOL_ERROR',
66
- message: `computer_open: \`${field}\` must not begin with '-' (would be parsed as an option flag)`,
67
- context: { tool: 'computer_open', field },
68
- });
69
- }
70
- }
71
- const args: string[] = [];
72
- if (app) {
73
- args.push('-a', app);
74
- }
75
- // `--` ends option parsing so the positional target is treated strictly
76
- // as a path/URL operand even if a future change relaxes the guard above.
77
- if (target) {
78
- args.push('--', target);
79
- }
80
- const proc = await runProcess('open', args, {
81
- ...(ctx.signal ? { signal: ctx.signal } : {}),
82
- timeoutMs: 10_000,
83
- });
84
- if (proc.exitCode !== 0) {
85
- const cause = procFailureCause(proc, 10_000);
86
- throw new MoxxyError({
87
- code: 'TOOL_ERROR',
88
- message: cause
89
- ? `open ${cause}`
90
- : `open failed (exit ${proc.exitCode}): ${proc.stderr.trim() || '(no error message)'}`,
91
- context: { tool: 'computer_open', exitCode: proc.exitCode, timedOut: proc.timedOut ? 1 : 0 },
92
- });
93
- }
94
- return { ok: true, app, target };
95
- },
96
- });
@@ -1,137 +0,0 @@
1
- import { promises as fs } from 'node:fs';
2
- import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
3
- import { MoxxyError } from '@moxxy/sdk';
4
-
5
- /**
6
- * Hermetic cover for the screenshot handler.
7
- *
8
- * The previous test shelled out to the real `screencapture`, which made it
9
- * depend on the host being macOS, having a display, and having been granted
10
- * Screen Recording. Worse, it accepted a thrown MoxxyError as a pass, so on any
11
- * machine without that permission (every CI runner) it verified nothing while
12
- * still reporting green. Faking the process layer means the contract below is
13
- * actually asserted, on every platform.
14
- */
15
- const runProcess = vi.fn();
16
- const ensureDarwin = vi.fn();
17
-
18
- vi.mock('../shell.js', async (importOriginal) => {
19
- const actual = await importOriginal<typeof import('../shell.js')>();
20
- return {
21
- ...actual,
22
- ensureDarwin: (...args: unknown[]) => ensureDarwin(...args),
23
- runProcess: (...args: unknown[]) => runProcess(...args),
24
- };
25
- });
26
-
27
- const { screenshotTool } = await import('./screenshot.js');
28
-
29
- const ctx = { signal: new AbortController().signal } as never;
30
-
31
- /** Smallest thing that is unmistakably PNG bytes; content is never decoded. */
32
- const PNG = Buffer.from('89504e470d0a1a0a0000000d49484452', 'hex');
33
-
34
- const ok = { exitCode: 0, stdout: '', stderr: '', timedOut: false };
35
-
36
- /** Track every temp path either command was pointed at, to assert cleanup. */
37
- let touched: string[] = [];
38
-
39
- const workingTools = (outBytes: Buffer = PNG): void => {
40
- runProcess.mockImplementation(async (cmd: string, args: string[]) => {
41
- if (cmd === 'screencapture') {
42
- const target = args[args.length - 1]!;
43
- touched.push(target);
44
- await fs.writeFile(target, PNG);
45
- return ok;
46
- }
47
- if (cmd === 'sips') {
48
- const out = args[args.indexOf('--out') + 1]!;
49
- touched.push(out);
50
- await fs.writeFile(out, outBytes);
51
- return ok;
52
- }
53
- throw new Error(`unexpected command ${cmd}`);
54
- });
55
- };
56
-
57
- beforeEach(() => {
58
- touched = [];
59
- ensureDarwin.mockReset();
60
- runProcess.mockReset();
61
- });
62
- afterEach(async () => {
63
- for (const f of touched) await fs.rm(f, { force: true });
64
- });
65
-
66
- describe('computer_screenshot', () => {
67
- it('returns { mediaType, base64 }, the shape the model can actually see', async () => {
68
- // Load-bearing: the SDK emits a provider `image` block only for exactly
69
- // this shape. Anything else is JSON.stringify'd and reaches the model as
70
- // base64 TEXT it cannot decode.
71
- workingTools();
72
-
73
- const result = (await screenshotTool.handler({}, ctx)) as Record<string, unknown>;
74
-
75
- expect(typeof result).toBe('object');
76
- // jpeg is the default: much smaller, and the model reads pixels not bytes.
77
- expect(result.mediaType).toBe('image/jpeg');
78
- expect(typeof result.base64).toBe('string');
79
- expect(result.base64).toBe(PNG.toString('base64'));
80
- });
81
-
82
- it('reports image/png when png was asked for', async () => {
83
- workingTools();
84
-
85
- const result = (await screenshotTool.handler({ format: 'png' }, ctx)) as Record<
86
- string,
87
- unknown
88
- >;
89
-
90
- expect(result.mediaType).toBe('image/png');
91
- });
92
-
93
- it('passes a region through to screencapture as -R', async () => {
94
- workingTools();
95
-
96
- await screenshotTool.handler({ region: { x: 1, y: 2, width: 3, height: 4 } }, ctx);
97
-
98
- const args = runProcess.mock.calls.find((c) => c[0] === 'screencapture')![1] as string[];
99
- expect(args[args.indexOf('-R') + 1]).toBe('1,2,3,4');
100
- });
101
-
102
- it('refuses on a non-darwin host rather than shelling out anyway', async () => {
103
- ensureDarwin.mockImplementation(() => {
104
- throw new MoxxyError({ code: 'TOOL_ERROR', message: 'darwin only' });
105
- });
106
-
107
- await expect(screenshotTool.handler({}, ctx)).rejects.toBeInstanceOf(MoxxyError);
108
- expect(runProcess).not.toHaveBeenCalled();
109
- });
110
-
111
- it('surfaces a capture failure as a MoxxyError, naming the likely cause', async () => {
112
- runProcess.mockResolvedValue({ exitCode: 1, stdout: '', stderr: '', timedOut: false });
113
-
114
- await expect(screenshotTool.handler({}, ctx)).rejects.toThrow(/screencapture failed/);
115
- });
116
-
117
- it('refuses an image that is still over the cap after compression', async () => {
118
- // Rather than hand the model megabytes it cannot use, say what to lower.
119
- workingTools(Buffer.alloc(6 * 1024 * 1024, 1));
120
-
121
- await expect(screenshotTool.handler({}, ctx)).rejects.toThrow(/exceeded/);
122
- });
123
-
124
- it('leaves no temp file behind, on success or on failure', async () => {
125
- workingTools();
126
- await screenshotTool.handler({}, ctx);
127
-
128
- runProcess.mockReset();
129
- workingTools(Buffer.alloc(6 * 1024 * 1024, 1));
130
- await screenshotTool.handler({}, ctx).catch(() => undefined);
131
-
132
- expect(touched.length).toBeGreaterThan(0);
133
- for (const f of touched) {
134
- await expect(fs.access(f)).rejects.toThrow();
135
- }
136
- });
137
- });
@@ -1,180 +0,0 @@
1
- import { randomUUID } from 'node:crypto';
2
- import { promises as fs } from 'node:fs';
3
- import * as os from 'node:os';
4
- import * as path from 'node:path';
5
- import { defineTool, MoxxyError, z } from '@moxxy/sdk';
6
- import { ensureDarwin, procFailureCause, runProcess } from '../shell.js';
7
- import { withTemporaryFiles } from '../temporary-files.js';
8
-
9
- const regionSchema = z.object({
10
- x: z.number().int().min(0),
11
- y: z.number().int().min(0),
12
- width: z.number().int().positive(),
13
- height: z.number().int().positive(),
14
- });
15
-
16
- /**
17
- * Defaults aggressive on size — a Retina full-screen PNG can pass
18
- * 5 MB base64-encoded and instantly blow the model's context window.
19
- * 1280px JPEG @ q72 is ~150 KB for a typical desktop and still
20
- * legible enough for the model to identify UI elements.
21
- */
22
- const DEFAULT_MAX_DIM = 1280;
23
- const DEFAULT_FORMAT = 'jpeg' as const;
24
- const DEFAULT_JPEG_QUALITY = 72;
25
- /** Soft cap on the encoded image size in bytes. We refuse to return
26
- * anything bigger than this even after the model asks for high-res —
27
- * better to fail loudly than silently nuke the context. */
28
- const MAX_BYTES = 1_500_000;
29
-
30
- export const screenshotTool = defineTool({
31
- name: 'computer_screenshot',
32
- icon: 'workspace',
33
- description:
34
- 'Take a screenshot of the macOS desktop. Returns base64-encoded image bytes ' +
35
- 'the model can see directly. By default the image is downscaled to 1280px ' +
36
- 'and JPEG-compressed so a single screenshot stays well under context — ' +
37
- 'override `maxDim` / `format` / `quality` only when you genuinely need full ' +
38
- 'resolution. macOS will prompt for Screen Recording permission on first use.',
39
- inputSchema: z.object({
40
- region: regionSchema
41
- .optional()
42
- .describe(
43
- 'Crop to a rectangle in screen-pixel coordinates (top-left origin). Omit for full screen.',
44
- ),
45
- maxDim: z
46
- .number()
47
- .int()
48
- .min(256)
49
- .max(3840)
50
- .optional()
51
- .describe(
52
- `Resize so the longest edge is <= this many pixels. Default ${DEFAULT_MAX_DIM}. ` +
53
- 'Lower = smaller payload + less context cost; raise only when you need pixel detail.',
54
- ),
55
- format: z
56
- .enum(['jpeg', 'png'])
57
- .optional()
58
- .describe(
59
- `Output format. Default "${DEFAULT_FORMAT}" (much smaller). Pick "png" only when ` +
60
- 'you need lossless (rare — JPEG is fine for UI screenshots).',
61
- ),
62
- quality: z
63
- .number()
64
- .int()
65
- .min(40)
66
- .max(100)
67
- .optional()
68
- .describe(
69
- `JPEG quality (1-100). Default ${DEFAULT_JPEG_QUALITY}. Ignored for PNG output.`,
70
- ),
71
- }),
72
- permission: { action: 'prompt' },
73
- // Capture + resize round-trips through temp files under os.tmpdir()
74
- // (which is /var/folders/... on macOS, not /tmp). Budget covers the two
75
- // 15s child stages plus encode/cleanup.
76
- isolation: {
77
- capabilities: {
78
- subprocess: true,
79
- commands: ['screencapture', 'sips'],
80
- fs: { read: [`${os.tmpdir()}/**`], write: [`${os.tmpdir()}/**`] },
81
- net: { mode: 'none' },
82
- timeMs: 45_000,
83
- },
84
- },
85
- async handler({ region, maxDim, format, quality }, ctx) {
86
- ensureDarwin('computer_screenshot');
87
- const fmt = format ?? DEFAULT_FORMAT;
88
- const dim = maxDim ?? DEFAULT_MAX_DIM;
89
- const q = quality ?? DEFAULT_JPEG_QUALITY;
90
-
91
- // Always capture as PNG first (preserves color depth + no double
92
- // compression), then let `sips` resize and convert in one pass.
93
- // `-x` silences the camera-shutter sound.
94
- // A random suffix guarantees uniqueness even for two captures in the
95
- // same process within the same millisecond (pid+Date.now() alone can
96
- // collide and cross-corrupt the temp files).
97
- const uniq = randomUUID();
98
- const captureTmp = path.join(
99
- os.tmpdir(),
100
- `moxxy-screencap-${process.pid}-${Date.now()}-${uniq}.png`,
101
- );
102
- const captureArgs = ['-x', '-t', 'png'];
103
- if (region) {
104
- captureArgs.push('-R', `${region.x},${region.y},${region.width},${region.height}`);
105
- }
106
- captureArgs.push(captureTmp);
107
- // Guarantee the original PNG (potentially several MB for a Retina full
108
- // screen) is removed on EVERY exit path — success, throw, or a spawn
109
- // reject (e.g. `sips` not on PATH) / mid-capture timeout that may have
110
- // left a partial file. Without this, those failure paths leak the temp
111
- // file in os.tmpdir() permanently and accumulate over repeated failures.
112
- const outExt = fmt === 'jpeg' ? 'jpg' : 'png';
113
- const outTmp = path.join(os.tmpdir(), `moxxy-screencap-${process.pid}-${uniq}-out.${outExt}`);
114
- return withTemporaryFiles([captureTmp, outTmp], async () => {
115
- const cap = await runProcess('screencapture', captureArgs, {
116
- ...(ctx.signal ? { signal: ctx.signal } : {}),
117
- timeoutMs: 15_000,
118
- });
119
- if (cap.exitCode !== 0) {
120
- const cause = procFailureCause(cap, 15_000);
121
- throw new MoxxyError({
122
- code: 'TOOL_ERROR',
123
- message: cause
124
- ? `screencapture ${cause}`
125
- : `screencapture failed (exit ${cap.exitCode}): ${cap.stderr.trim() || '(no stderr — likely Screen Recording permission missing — grant in System Settings → Privacy & Security)'}`,
126
- context: { tool: 'computer_screenshot', exitCode: cap.exitCode, timedOut: cap.timedOut ? 1 : 0 },
127
- });
128
- }
129
-
130
- // Resize + format-convert in one sips call. `-Z N` fits within N
131
- // on the longest edge while preserving aspect ratio. Output ext
132
- // picks the format; format options apply when JPEG.
133
- const sipsArgs = [
134
- '-Z',
135
- String(dim),
136
- '--setProperty',
137
- 'format',
138
- fmt,
139
- ];
140
- if (fmt === 'jpeg') {
141
- sipsArgs.push('--setProperty', 'formatOptions', String(q));
142
- }
143
- sipsArgs.push(captureTmp, '--out', outTmp);
144
- const sip = await runProcess('sips', sipsArgs, {
145
- ...(ctx.signal ? { signal: ctx.signal } : {}),
146
- timeoutMs: 15_000,
147
- });
148
- if (sip.exitCode !== 0) {
149
- const cause = procFailureCause(sip, 15_000);
150
- throw new MoxxyError({
151
- code: 'TOOL_ERROR',
152
- message: cause
153
- ? `sips resize/convert ${cause}`
154
- : `sips resize/convert failed (exit ${sip.exitCode}): ${sip.stderr.trim() || '(no error message)'}`,
155
- context: { tool: 'computer_screenshot', exitCode: sip.exitCode, timedOut: sip.timedOut ? 1 : 0 },
156
- });
157
- }
158
-
159
- const bytes = await fs.readFile(outTmp);
160
- if (bytes.length > MAX_BYTES) {
161
- throw new MoxxyError({
162
- code: 'TOOL_ERROR',
163
- message:
164
- `screenshot exceeded ${MAX_BYTES} bytes after compression (got ${bytes.length}). ` +
165
- `Lower maxDim (currently ${dim}) or quality (currently ${q}), or pass a smaller region.`,
166
- context: { tool: 'computer_screenshot', byteLength: bytes.length },
167
- });
168
- }
169
- // Preserve the image-shaped output consumed by the SDK projection.
170
- return {
171
- mediaType: fmt === 'jpeg' ? ('image/jpeg' as const) : ('image/png' as const),
172
- base64: bytes.toString('base64'),
173
- byteLength: bytes.length,
174
- maxDim: dim,
175
- format: fmt,
176
- ...(fmt === 'jpeg' ? { quality: q } : {}),
177
- };
178
- });
179
- },
180
- });
package/src/tools/type.ts DELETED
@@ -1,68 +0,0 @@
1
- import { defineTool, MoxxyError, z } from '@moxxy/sdk';
2
- import { ensureDarwin, procFailureCause, runProcess } from '../shell.js';
3
-
4
- export const typeTool = defineTool({
5
- name: 'computer_type',
6
- description:
7
- 'Type a string into whatever has keyboard focus. Use AFTER clicking the ' +
8
- 'target field. Macros (cmd+C, etc.) belong in computer_key, not here — this ' +
9
- 'tool sends each character literally. Requires Accessibility permission.',
10
- inputSchema: z.object({
11
- text: z
12
- .string()
13
- .max(4000)
14
- .describe(
15
- 'The literal text to type. Newlines and tabs are typed as-is. ' +
16
- 'Pre-existing focus is the target — click first if needed.',
17
- ),
18
- }),
19
- permission: { action: 'prompt' },
20
- isolation: {
21
- capabilities: {
22
- subprocess: true,
23
- commands: ['osascript'],
24
- net: { mode: 'none' },
25
- timeMs: 40_000,
26
- },
27
- },
28
- async handler({ text }, ctx) {
29
- ensureDarwin('computer_type');
30
- if (text.length === 0) return { ok: true, length: 0 };
31
- // Serialize the text to an AppleScript string literal ourselves and
32
- // pass it inline via `osascript -e`, so we never have to escape
33
- // AppleScript quote rules at the shell layer.
34
- const literal = toAppleScriptString(text);
35
- const directScript = `tell application "System Events" to keystroke ${literal}`;
36
- const proc = await runProcess('osascript', ['-e', directScript], {
37
- ...(ctx.signal ? { signal: ctx.signal } : {}),
38
- timeoutMs: 30_000,
39
- });
40
- if (proc.exitCode !== 0) {
41
- const cause = procFailureCause(proc, 30_000);
42
- throw new MoxxyError({
43
- code: 'TOOL_ERROR',
44
- message: cause
45
- ? `type ${cause}`
46
- : `type failed (exit ${proc.exitCode}): ${proc.stderr.trim() || '(check Accessibility permission)'}`,
47
- context: { tool: 'computer_type', exitCode: proc.exitCode, timedOut: proc.timedOut ? 1 : 0 },
48
- });
49
- }
50
- return { ok: true, length: text.length };
51
- },
52
- });
53
-
54
- /**
55
- * Serialize an arbitrary JS string into a valid AppleScript string
56
- * literal. AppleScript string syntax: `"..."` with `\"` and `\\` as
57
- * the only escapes; newlines are written as `" & return & "`.
58
- */
59
- export function toAppleScriptString(s: string): string {
60
- // Split on newlines to use `return` (AppleScript's CR constant) so
61
- // a multi-line type call sends actual Enter keystrokes rather than a
62
- // literal `\n` in one keystroke (which keystroke would refuse).
63
- const parts = s.split('\n').map((line) => {
64
- const escaped = line.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
65
- return `"${escaped}"`;
66
- });
67
- return parts.join(' & return & ');
68
- }
package/src/tools.test.ts DELETED
@@ -1,94 +0,0 @@
1
- import { MoxxyError } from '@moxxy/sdk';
2
- import { describe, expect, it } from 'vitest';
3
- import {
4
- applescriptTool,
5
- clickTool,
6
- clipboardTool,
7
- keyTool,
8
- openTool,
9
- screenshotTool,
10
- typeTool,
11
- } from './index.js';
12
-
13
- /** Minimal ToolContext stub — only `signal` is read by these handlers. */
14
- const fakeCtx = { signal: new AbortController().signal } as never;
15
-
16
- describe('@moxxy/plugin-computer-control schemas', () => {
17
- it('screenshot accepts an optional region', () => {
18
- expect(screenshotTool.inputSchema.safeParse({}).success).toBe(true);
19
- expect(
20
- screenshotTool.inputSchema.safeParse({ region: { x: 0, y: 0, width: 100, height: 100 } })
21
- .success,
22
- ).toBe(true);
23
- });
24
-
25
- it('click requires x and y, count 1-3 only', () => {
26
- expect(clickTool.inputSchema.safeParse({ x: 100, y: 200 }).success).toBe(true);
27
- expect(clickTool.inputSchema.safeParse({ x: 100, y: 200, count: 2 }).success).toBe(true);
28
- expect(clickTool.inputSchema.safeParse({ x: 100, y: 200, count: 5 }).success).toBe(false);
29
- expect(clickTool.inputSchema.safeParse({ x: -1, y: 0 }).success).toBe(false);
30
- });
31
-
32
- it('type rejects oversize payloads', () => {
33
- expect(typeTool.inputSchema.safeParse({ text: 'hello' }).success).toBe(true);
34
- expect(typeTool.inputSchema.safeParse({ text: 'a'.repeat(5000) }).success).toBe(false);
35
- });
36
-
37
- it('key allows known modifier names and rejects unknown ones', () => {
38
- expect(keyTool.inputSchema.safeParse({ key: 'tab' }).success).toBe(true);
39
- expect(
40
- keyTool.inputSchema.safeParse({ key: 'a', modifiers: ['cmd', 'shift'] }).success,
41
- ).toBe(true);
42
- expect(
43
- keyTool.inputSchema.safeParse({ key: 'a', modifiers: ['windows'] }).success,
44
- ).toBe(false);
45
- });
46
-
47
- it('open requires target or app at runtime (not enforced by zod)', () => {
48
- // Both optional at schema level — runtime guard does the validation.
49
- expect(openTool.inputSchema.safeParse({ app: 'Safari' }).success).toBe(true);
50
- expect(openTool.inputSchema.safeParse({ target: '/tmp' }).success).toBe(true);
51
- expect(openTool.inputSchema.safeParse({}).success).toBe(true);
52
- });
53
-
54
- it('clipboard requires action enum', () => {
55
- expect(clipboardTool.inputSchema.safeParse({ action: 'read' }).success).toBe(true);
56
- expect(clipboardTool.inputSchema.safeParse({ action: 'write', text: 'x' }).success).toBe(true);
57
- expect(clipboardTool.inputSchema.safeParse({ action: 'paste' }).success).toBe(false);
58
- });
59
-
60
- it('applescript needs a non-empty script', () => {
61
- expect(applescriptTool.inputSchema.safeParse({ script: 'return 1' }).success).toBe(true);
62
- expect(applescriptTool.inputSchema.safeParse({ script: '' }).success).toBe(false);
63
- });
64
-
65
- it('open rejects a target/app beginning with "-" (argument-injection guard, darwin only)', async () => {
66
- // The guard runs after ensureDarwin, so on non-darwin hosts ensureDarwin
67
- // throws first — assert that path too so the test is meaningful everywhere.
68
- if (process.platform === 'darwin') {
69
- await expect(openTool.handler({ target: '-g' }, fakeCtx)).rejects.toThrow(
70
- /must not begin with '-'/,
71
- );
72
- await expect(openTool.handler({ app: '-n' }, fakeCtx)).rejects.toThrow(
73
- /must not begin with '-'/,
74
- );
75
- } else {
76
- await expect(openTool.handler({ target: '-g' }, fakeCtx)).rejects.toBeInstanceOf(MoxxyError);
77
- }
78
- });
79
-
80
- it('every tool is permission:prompt — never auto-allowed', () => {
81
- const all = [
82
- screenshotTool,
83
- clickTool,
84
- typeTool,
85
- keyTool,
86
- openTool,
87
- clipboardTool,
88
- applescriptTool,
89
- ];
90
- for (const tool of all) {
91
- expect(tool.permission?.action).toBe('prompt');
92
- }
93
- });
94
- });
@@ -1,13 +0,0 @@
1
- import { expect, it } from 'vitest';
2
- import { actionSchema, actionStatusSchema, actionResultSchema } from './contracts.js';
3
-
4
- it('only permits explicit observed UIA actions and bounded receipt waiting', () => {
5
- const target={windowId:'w',observationId:'o',elementId:'e'};
6
- expect(actionSchema.safeParse({...target,action:'toggle'}).success).toBe(true);
7
- expect(actionSchema.safeParse({...target,action:'eval'}).success).toBe(false);
8
- expect(actionSchema.safeParse({...target,action:'invoke',fallback:'click'}).success).toBe(false);
9
- expect(actionStatusSchema.parse({actionId:'a'})).toEqual({actionId:'a',waitMs:0});
10
- expect(actionStatusSchema.safeParse({actionId:'a',waitMs:1001}).success).toBe(false);
11
- expect(actionResultSchema.safeParse({actionId:'a',status:'pending',verificationRequired:true}).success).toBe(true);
12
- expect(actionResultSchema.safeParse({actionId:'a',status:'pending',verificationRequired:false}).success).toBe(false);
13
- });
@@ -1,16 +0,0 @@
1
- import { createHash } from 'node:crypto';
2
- import { expect, it } from 'vitest';
3
- import { validateHelperArtifact } from './artifact.js';
4
-
5
- it('validates PE architecture, protocol and exact executable digest before launch', () => {
6
- // Synthetic file header tests the parser, not native execution.
7
- const bytes = Buffer.alloc(256);
8
- bytes.write('MZ'); bytes.writeUInt32LE(128, 60); bytes.write('PE\0\0', 128); bytes.writeUInt16LE(0x8664, 132);
9
- const manifest = { protocolVersion: 4, architecture: 'x64', sha256: createHash('sha256').update(bytes).digest('hex') };
10
- expect(() => validateHelperArtifact(bytes, manifest)).not.toThrow();
11
- expect(() => validateHelperArtifact(bytes, { ...manifest, protocolVersion: 1 })).toThrow();
12
- expect(() => validateHelperArtifact(bytes, { ...manifest, sha256: '0'.repeat(64) })).toThrow();
13
- bytes.writeUInt16LE(0xAA64, 132);
14
- expect(() => validateHelperArtifact(bytes, { ...manifest, sha256: createHash('sha256').update(bytes).digest('hex') })).toThrow(/x64/);
15
- expect(() => validateHelperArtifact(Buffer.alloc(2), manifest)).toThrow();
16
- });
@@ -1,57 +0,0 @@
1
- import { createHash } from 'node:crypto';
2
- import { constants } from 'node:fs';
3
- import { lstat, open } from 'node:fs/promises';
4
- import type { FileHandle } from 'node:fs/promises';
5
- import { z } from 'zod';
6
- import { PROTOCOL_VERSION } from './contracts.js';
7
-
8
- const manifestSchema = z.object({ protocolVersion: z.literal(PROTOCOL_VERSION), architecture: z.literal('x64'), sha256: z.string().regex(/^[a-f0-9]{64}$/) }).strict();
9
-
10
- export function validateHelperArtifact(bytes: Buffer, manifest: unknown): void {
11
- const expected = manifestSchema.parse(manifest);
12
- if (bytes.length < 64 || bytes.toString('ascii', 0, 2) !== 'MZ') throw new Error('Invalid Computer Use executable');
13
- const offset = bytes.readUInt32LE(60);
14
- if (offset > bytes.length - 6 || bytes.toString('ascii', offset, offset + 4) !== 'PE\0\0' || bytes.readUInt16LE(offset + 4) !== 0x8664) {
15
- throw new Error('Computer Use requires a Windows x64 executable');
16
- }
17
- if (createHash('sha256').update(bytes).digest('hex') !== expected.sha256) throw new Error('Computer Use executable checksum mismatch');
18
- }
19
-
20
- // POSIX-only; Node leaves it undefined on Windows.
21
- const O_NOFOLLOW = constants.O_NOFOLLOW ?? 0;
22
-
23
- /**
24
- * Read a regular file no larger than `limit`, checking the open handle rather
25
- * than the path so the bytes that are validated are the bytes that were
26
- * measured. A `stat` followed by `readFile` can be repointed in between, which
27
- * would let an oversized file through the size guard.
28
- */
29
- async function readBounded(file: string, limit: number): Promise<Buffer> {
30
- if (O_NOFOLLOW === 0 && (await lstat(file)).isSymbolicLink()) {
31
- throw new Error('Computer Use artifact exceeds size limit');
32
- }
33
- let handle: FileHandle | undefined;
34
- try {
35
- handle = await open(file, constants.O_RDONLY | O_NOFOLLOW);
36
- const info = await handle.stat();
37
- if (!info.isFile() || info.size > limit) throw new Error('Computer Use artifact exceeds size limit');
38
- return await handle.readFile();
39
- } catch (error) {
40
- // O_NOFOLLOW reports a symlink as ELOOP.
41
- if ((error as NodeJS.ErrnoException).code === 'ELOOP') {
42
- throw new Error('Computer Use artifact exceeds size limit');
43
- }
44
- throw error;
45
- } finally {
46
- await handle?.close();
47
- }
48
- }
49
-
50
- export async function verifyHelperArtifact(executable: string): Promise<void> {
51
- const manifestPath = executable + '.json';
52
- const [bytes, manifest] = await Promise.all([
53
- readBounded(executable, 32_000_000),
54
- readBounded(manifestPath, 4096),
55
- ]);
56
- validateHelperArtifact(bytes, JSON.parse(manifest.toString('utf8')));
57
- }