@moxxy/plugin-computer-control 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/LICENSE +21 -0
  2. package/dist/index.d.ts +30 -0
  3. package/dist/index.d.ts.map +1 -0
  4. package/dist/index.js +43 -0
  5. package/dist/index.js.map +1 -0
  6. package/dist/shell.d.ts +56 -0
  7. package/dist/shell.d.ts.map +1 -0
  8. package/dist/shell.js +189 -0
  9. package/dist/shell.js.map +1 -0
  10. package/dist/tools/applescript.d.ts +2 -0
  11. package/dist/tools/applescript.d.ts.map +1 -0
  12. package/dist/tools/applescript.js +38 -0
  13. package/dist/tools/applescript.js.map +1 -0
  14. package/dist/tools/click.d.ts +2 -0
  15. package/dist/tools/click.d.ts.map +1 -0
  16. package/dist/tools/click.js +50 -0
  17. package/dist/tools/click.js.map +1 -0
  18. package/dist/tools/clipboard.d.ts +2 -0
  19. package/dist/tools/clipboard.d.ts.map +1 -0
  20. package/dist/tools/clipboard.js +62 -0
  21. package/dist/tools/clipboard.js.map +1 -0
  22. package/dist/tools/key.d.ts +11 -0
  23. package/dist/tools/key.d.ts.map +1 -0
  24. package/dist/tools/key.js +135 -0
  25. package/dist/tools/key.js.map +1 -0
  26. package/dist/tools/open.d.ts +2 -0
  27. package/dist/tools/open.d.ts.map +1 -0
  28. package/dist/tools/open.js +76 -0
  29. package/dist/tools/open.js.map +1 -0
  30. package/dist/tools/screenshot.d.ts +2 -0
  31. package/dist/tools/screenshot.d.ts.map +1 -0
  32. package/dist/tools/screenshot.js +162 -0
  33. package/dist/tools/screenshot.js.map +1 -0
  34. package/dist/tools/type.d.ts +8 -0
  35. package/dist/tools/type.d.ts.map +1 -0
  36. package/dist/tools/type.js +57 -0
  37. package/dist/tools/type.js.map +1 -0
  38. package/package.json +65 -0
  39. package/skills/computer-control.md +159 -0
  40. package/src/index.ts +55 -0
  41. package/src/shell.test.ts +186 -0
  42. package/src/shell.ts +213 -0
  43. package/src/tools/applescript-serialize.test.ts +74 -0
  44. package/src/tools/applescript.ts +41 -0
  45. package/src/tools/click.ts +52 -0
  46. package/src/tools/clipboard.ts +63 -0
  47. package/src/tools/key.ts +147 -0
  48. package/src/tools/open.ts +81 -0
  49. package/src/tools/screenshot.ts +181 -0
  50. package/src/tools/type.ts +60 -0
  51. package/src/tools.test.ts +128 -0
@@ -0,0 +1,41 @@
1
+ import { defineTool, MoxxyError, z } from '@moxxy/sdk';
2
+ import { ensureDarwin, procFailureCause, runProcess } from '../shell.js';
3
+
4
+ export const applescriptTool = defineTool({
5
+ name: 'computer_applescript',
6
+ description:
7
+ 'Escape hatch: run an arbitrary AppleScript snippet via `osascript`. Use ' +
8
+ 'for anything the dedicated tools can\'t cover (frontmost app name, list ' +
9
+ 'open Safari tabs, drive a specific app via its scripting dictionary, etc.). ' +
10
+ 'Returns the script\'s stdout. The user approves every invocation — keep ' +
11
+ 'scripts focused and reversible.',
12
+ inputSchema: z.object({
13
+ script: z
14
+ .string()
15
+ .min(1)
16
+ .max(8000)
17
+ .describe(
18
+ 'AppleScript source. Multi-line is fine — pass `\\n` between statements. ' +
19
+ 'For JavaScript for Automation, prefix with `#!/usr/bin/osascript -l JavaScript`',
20
+ ),
21
+ }),
22
+ permission: { action: 'prompt' },
23
+ async handler({ script }, ctx) {
24
+ ensureDarwin('computer_applescript');
25
+ const proc = await runProcess('osascript', ['-e', script], {
26
+ ...(ctx.signal ? { signal: ctx.signal } : {}),
27
+ timeoutMs: 30_000,
28
+ });
29
+ if (proc.exitCode !== 0) {
30
+ const cause = procFailureCause(proc, 30_000);
31
+ throw new MoxxyError({
32
+ code: 'TOOL_ERROR',
33
+ message: cause
34
+ ? `osascript ${cause}`
35
+ : `osascript failed (exit ${proc.exitCode}): ${proc.stderr.trim() || '(no error message)'}`,
36
+ context: { tool: 'computer_applescript', exitCode: proc.exitCode, timedOut: proc.timedOut ? 1 : 0 },
37
+ });
38
+ }
39
+ return { ok: true, output: proc.stdout.trim() };
40
+ },
41
+ });
@@ -0,0 +1,52 @@
1
+ import { defineTool, MoxxyError, z } from '@moxxy/sdk';
2
+ import { ensureDarwin, procFailureCause, runProcess } from '../shell.js';
3
+
4
+ export const clickTool = defineTool({
5
+ name: 'computer_click',
6
+ description:
7
+ 'Click the mouse at screen-pixel coordinates (top-left origin). Uses macOS ' +
8
+ 'System Events; requires Accessibility permission on first use. Pass count=2 ' +
9
+ 'for a double-click. Right/middle buttons are not supported via this tool — ' +
10
+ 'use computer_applescript for those (rare).',
11
+ inputSchema: z.object({
12
+ x: z.number().int().min(0).describe('X pixel from the top-left of the display.'),
13
+ y: z.number().int().min(0).describe('Y pixel from the top-left of the display.'),
14
+ count: z
15
+ .number()
16
+ .int()
17
+ .min(1)
18
+ .max(3)
19
+ .optional()
20
+ .describe('Number of consecutive clicks. 1 = single, 2 = double, 3 = triple. Default 1.'),
21
+ }),
22
+ permission: { action: 'prompt' },
23
+ async handler({ x, y, count }, ctx) {
24
+ ensureDarwin('computer_click');
25
+ const n = count ?? 1;
26
+ // `click at {x, y}` repeats N times via a repeat block. Use a
27
+ // bare AppleScript string with numeric interpolation — no
28
+ // user-supplied text reaches the script, so injection isn't a
29
+ // risk here.
30
+ const script =
31
+ `tell application "System Events"\n` +
32
+ ` repeat ${n} times\n` +
33
+ ` click at {${Math.round(x)}, ${Math.round(y)}}\n` +
34
+ ` end repeat\n` +
35
+ `end tell`;
36
+ const proc = await runProcess('osascript', ['-e', script], {
37
+ ...(ctx.signal ? { signal: ctx.signal } : {}),
38
+ timeoutMs: 10_000,
39
+ });
40
+ if (proc.exitCode !== 0) {
41
+ const cause = procFailureCause(proc, 10_000);
42
+ throw new MoxxyError({
43
+ code: 'TOOL_ERROR',
44
+ message: cause
45
+ ? `click ${cause}`
46
+ : `click failed (exit ${proc.exitCode}): ${proc.stderr.trim() || '(check Accessibility permission in System Settings → Privacy & Security → Accessibility)'}`,
47
+ context: { tool: 'computer_click', exitCode: proc.exitCode, timedOut: proc.timedOut ? 1 : 0 },
48
+ });
49
+ }
50
+ return { ok: true, x, y, count: n };
51
+ },
52
+ });
@@ -0,0 +1,63 @@
1
+ import { defineTool, MoxxyError, z } from '@moxxy/sdk';
2
+ import { ensureDarwin, procFailureCause, runProcess } from '../shell.js';
3
+
4
+ export const clipboardTool = defineTool({
5
+ name: 'computer_clipboard',
6
+ description:
7
+ 'Read from or write to the macOS clipboard. Use `read` to fetch what the ' +
8
+ 'user just copied; use `write` to stage text the user can then paste with ' +
9
+ '⌘V. Writing the clipboard does NOT trigger a paste — call computer_key ' +
10
+ 'with cmd+v after if you need to paste.',
11
+ inputSchema: z.object({
12
+ action: z.enum(['read', 'write']),
13
+ text: z
14
+ .string()
15
+ .max(64_000)
16
+ .optional()
17
+ .describe('Text to put on the clipboard. Required when action="write".'),
18
+ }),
19
+ permission: { action: 'prompt' },
20
+ async handler({ action, text }, ctx) {
21
+ ensureDarwin('computer_clipboard');
22
+ if (action === 'read') {
23
+ const proc = await runProcess('pbpaste', [], {
24
+ ...(ctx.signal ? { signal: ctx.signal } : {}),
25
+ timeoutMs: 5_000,
26
+ });
27
+ if (proc.exitCode !== 0) {
28
+ const cause = procFailureCause(proc, 5_000);
29
+ throw new MoxxyError({
30
+ code: 'TOOL_ERROR',
31
+ message: cause
32
+ ? `pbpaste ${cause}`
33
+ : `pbpaste failed (exit ${proc.exitCode}): ${proc.stderr.trim()}`,
34
+ context: { tool: 'computer_clipboard', exitCode: proc.exitCode, timedOut: proc.timedOut ? 1 : 0 },
35
+ });
36
+ }
37
+ return { ok: true, text: proc.stdout };
38
+ }
39
+ if (text === undefined) {
40
+ throw new MoxxyError({
41
+ code: 'TOOL_ERROR',
42
+ message: 'computer_clipboard: `text` is required when action="write"',
43
+ context: { tool: 'computer_clipboard' },
44
+ });
45
+ }
46
+ const proc = await runProcess('pbcopy', [], {
47
+ ...(ctx.signal ? { signal: ctx.signal } : {}),
48
+ input: text,
49
+ timeoutMs: 5_000,
50
+ });
51
+ if (proc.exitCode !== 0) {
52
+ const cause = procFailureCause(proc, 5_000);
53
+ throw new MoxxyError({
54
+ code: 'TOOL_ERROR',
55
+ message: cause
56
+ ? `pbcopy ${cause}`
57
+ : `pbcopy failed (exit ${proc.exitCode}): ${proc.stderr.trim()}`,
58
+ context: { tool: 'computer_clipboard', exitCode: proc.exitCode, timedOut: proc.timedOut ? 1 : 0 },
59
+ });
60
+ }
61
+ return { ok: true, length: text.length };
62
+ },
63
+ });
@@ -0,0 +1,147 @@
1
+ import { defineTool, MoxxyError, z } from '@moxxy/sdk';
2
+ import { ensureDarwin, procFailureCause, runProcess } from '../shell.js';
3
+
4
+ const MODIFIER_NAMES = ['cmd', 'shift', 'option', 'control'] as const;
5
+ type Modifier = (typeof MODIFIER_NAMES)[number];
6
+
7
+ /**
8
+ * Named keys → macOS key codes used by AppleScript's
9
+ * `key code N` command. Subset focused on what an automation agent
10
+ * actually needs (navigation, editing, function keys). Letters and
11
+ * digits go through `keystroke "x" using ...` instead of `key code`.
12
+ */
13
+ const KEY_CODES: Record<string, number> = {
14
+ return: 36,
15
+ enter: 36,
16
+ tab: 48,
17
+ space: 49,
18
+ escape: 53,
19
+ esc: 53,
20
+ delete: 51,
21
+ backspace: 51,
22
+ forward_delete: 117,
23
+ left: 123,
24
+ right: 124,
25
+ down: 125,
26
+ up: 126,
27
+ home: 115,
28
+ end: 119,
29
+ page_up: 116,
30
+ page_down: 121,
31
+ f1: 122,
32
+ f2: 120,
33
+ f3: 99,
34
+ f4: 118,
35
+ f5: 96,
36
+ f6: 97,
37
+ f7: 98,
38
+ f8: 100,
39
+ f9: 101,
40
+ f10: 109,
41
+ f11: 103,
42
+ f12: 111,
43
+ };
44
+
45
+ /**
46
+ * Single literal whitespace chars → their key codes. A bare ' '/'\t'/'\r'/'\n'
47
+ * would otherwise be sent via `keystroke`, which types the character (wrong for
48
+ * modifier chords like ctrl+space). Mirrors the matching KEY_CODES entries.
49
+ */
50
+ const WHITESPACE_KEY_CODES: Record<string, number> = {
51
+ ' ': 49, // space
52
+ '\t': 48, // tab
53
+ '\r': 36, // return
54
+ '\n': 36, // return
55
+ };
56
+
57
+ export const keyTool = defineTool({
58
+ name: 'computer_key',
59
+ description:
60
+ 'Send a single key chord with optional modifiers. Use this for shortcuts ' +
61
+ '(cmd+c, cmd+tab, cmd+shift+4) and named keys (return, tab, escape, arrows, ' +
62
+ 'page_up/down, f1–f12). For typing arbitrary text, use computer_type.',
63
+ inputSchema: z.object({
64
+ key: z
65
+ .string()
66
+ .min(1)
67
+ .describe(
68
+ 'A single character ("a", "1", "/") OR a named key from the catalog: ' +
69
+ Object.keys(KEY_CODES).join(', ') +
70
+ '.',
71
+ ),
72
+ modifiers: z
73
+ .array(z.enum(MODIFIER_NAMES))
74
+ .optional()
75
+ .describe(
76
+ 'Held modifiers. Common combos: ["cmd"] for cmd+key, ["cmd","shift"], ["control","option"].',
77
+ ),
78
+ }),
79
+ permission: { action: 'prompt' },
80
+ async handler({ key, modifiers }, ctx) {
81
+ ensureDarwin('computer_key');
82
+ const mods = modifiers ?? [];
83
+ const script = buildKeyScript(key, mods);
84
+ const proc = await runProcess('osascript', ['-e', script], {
85
+ ...(ctx.signal ? { signal: ctx.signal } : {}),
86
+ timeoutMs: 10_000,
87
+ });
88
+ if (proc.exitCode !== 0) {
89
+ const cause = procFailureCause(proc, 10_000);
90
+ throw new MoxxyError({
91
+ code: 'TOOL_ERROR',
92
+ message: cause
93
+ ? `key ${cause}`
94
+ : `key failed (exit ${proc.exitCode}): ${proc.stderr.trim() || '(check Accessibility permission)'}`,
95
+ context: { tool: 'computer_key', exitCode: proc.exitCode, timedOut: proc.timedOut ? 1 : 0 },
96
+ });
97
+ }
98
+ return { ok: true, key, modifiers: mods };
99
+ },
100
+ });
101
+
102
+ /**
103
+ * Build the `osascript -e` body for a single key chord. Named keys go
104
+ * through `key code N`, single characters through `keystroke "x"`, and a
105
+ * multi-char unknown key throws. Pure (no I/O) so it can be unit-tested.
106
+ */
107
+ export function buildKeyScript(key: string, modifiers: ReadonlyArray<Modifier>): string {
108
+ const usingClause =
109
+ modifiers.length > 0 ? ` using {${modifiers.map(modifierClause).join(', ')}}` : '';
110
+ const lower = key.toLowerCase();
111
+ if (lower in KEY_CODES) {
112
+ return `tell application "System Events" to key code ${KEY_CODES[lower]}${usingClause}`;
113
+ }
114
+ // A single literal whitespace char (' ', '\t', '\r', '\n') keystroked with a
115
+ // held modifier types the character rather than pressing the key, breaking
116
+ // chords like ctrl+space. Route it to the corresponding key code instead.
117
+ if (key.length === 1) {
118
+ const wsCode = WHITESPACE_KEY_CODES[key];
119
+ if (wsCode !== undefined) {
120
+ return `tell application "System Events" to key code ${wsCode}${usingClause}`;
121
+ }
122
+ }
123
+ if (key.length === 1) {
124
+ const literal = `"${key.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
125
+ return `tell application "System Events" to keystroke ${literal}${usingClause}`;
126
+ }
127
+ throw new MoxxyError({
128
+ code: 'TOOL_ERROR',
129
+ message: `unknown key "${key}". Use a single character or one of: ${Object.keys(KEY_CODES).join(', ')}.`,
130
+ context: { tool: 'computer_key', key },
131
+ });
132
+ }
133
+
134
+ function modifierClause(m: Modifier): string {
135
+ // AppleScript uses "command down", "shift down", "option down",
136
+ // "control down" — map our short names.
137
+ switch (m) {
138
+ case 'cmd':
139
+ return 'command down';
140
+ case 'shift':
141
+ return 'shift down';
142
+ case 'option':
143
+ return 'option down';
144
+ case 'control':
145
+ return 'control down';
146
+ }
147
+ }
@@ -0,0 +1,81 @@
1
+ import { defineTool, MoxxyError, z } from '@moxxy/sdk';
2
+ import { ensureDarwin, procFailureCause, runProcess } from '../shell.js';
3
+
4
+ export const openTool = defineTool({
5
+ name: 'computer_open',
6
+ description:
7
+ 'Open a URL, file path, or .app bundle via macOS `open`. Use this to ' +
8
+ 'launch / activate a specific app or jump to a web page. The model should ' +
9
+ 'prefer this over typing into Spotlight when the target is known.',
10
+ inputSchema: z.object({
11
+ target: z
12
+ .string()
13
+ .min(1)
14
+ .describe(
15
+ 'URL (https://...), file path (/Users/...), or app name (Safari). ' +
16
+ 'For app names, prefer the `app` field — `target` is treated as a path.',
17
+ )
18
+ .optional(),
19
+ app: z
20
+ .string()
21
+ .min(1)
22
+ .max(120)
23
+ .describe(
24
+ 'Application name to activate (e.g. "Safari", "Visual Studio Code"). ' +
25
+ 'When `target` is also set, the app opens `target` (e.g. open a file in VS Code).',
26
+ )
27
+ .optional(),
28
+ }),
29
+ permission: { action: 'prompt' },
30
+ async handler({ target, app }, ctx) {
31
+ ensureDarwin('computer_open');
32
+ if (!target && !app) {
33
+ throw new MoxxyError({
34
+ code: 'TOOL_ERROR',
35
+ message: 'computer_open: at least one of `target` or `app` is required',
36
+ context: { tool: 'computer_open' },
37
+ });
38
+ }
39
+ // A target/app beginning with '-' (e.g. '-g', '-n', '-W') is parsed by
40
+ // /usr/bin/open as an OPTION FLAG, not a path/URL/app — silently changing
41
+ // behavior (open in background, new instance) the user did not approve.
42
+ // Reject it: no legitimate path/URL/app name starts with a dash, and
43
+ // spawn is array-form so this is the only argument-injection vector.
44
+ for (const [field, value] of [
45
+ ['target', target],
46
+ ['app', app],
47
+ ] as const) {
48
+ if (value !== undefined && value.startsWith('-')) {
49
+ throw new MoxxyError({
50
+ code: 'TOOL_ERROR',
51
+ message: `computer_open: \`${field}\` must not begin with '-' (would be parsed as an option flag)`,
52
+ context: { tool: 'computer_open', field },
53
+ });
54
+ }
55
+ }
56
+ const args: string[] = [];
57
+ if (app) {
58
+ args.push('-a', app);
59
+ }
60
+ // `--` ends option parsing so the positional target is treated strictly
61
+ // as a path/URL operand even if a future change relaxes the guard above.
62
+ if (target) {
63
+ args.push('--', target);
64
+ }
65
+ const proc = await runProcess('open', args, {
66
+ ...(ctx.signal ? { signal: ctx.signal } : {}),
67
+ timeoutMs: 10_000,
68
+ });
69
+ if (proc.exitCode !== 0) {
70
+ const cause = procFailureCause(proc, 10_000);
71
+ throw new MoxxyError({
72
+ code: 'TOOL_ERROR',
73
+ message: cause
74
+ ? `open ${cause}`
75
+ : `open failed (exit ${proc.exitCode}): ${proc.stderr.trim() || '(no error message)'}`,
76
+ context: { tool: 'computer_open', exitCode: proc.exitCode, timedOut: proc.timedOut ? 1 : 0 },
77
+ });
78
+ }
79
+ return { ok: true, app, target };
80
+ },
81
+ });
@@ -0,0 +1,181 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import { promises as fs } from 'node:fs';
3
+ import * as os from 'node:os';
4
+ import * as path from 'node:path';
5
+ import { defineTool, MoxxyError, z } from '@moxxy/sdk';
6
+ import { ensureDarwin, procFailureCause, runProcess } from '../shell.js';
7
+
8
+ const regionSchema = z.object({
9
+ x: z.number().int().min(0),
10
+ y: z.number().int().min(0),
11
+ width: z.number().int().positive(),
12
+ height: z.number().int().positive(),
13
+ });
14
+
15
+ /**
16
+ * Defaults aggressive on size — a Retina full-screen PNG can pass
17
+ * 5 MB base64-encoded and instantly blow the model's context window.
18
+ * 1280px JPEG @ q72 is ~150 KB for a typical desktop and still
19
+ * legible enough for the model to identify UI elements.
20
+ */
21
+ const DEFAULT_MAX_DIM = 1280;
22
+ const DEFAULT_FORMAT = 'jpeg' as const;
23
+ const DEFAULT_JPEG_QUALITY = 72;
24
+ /** Soft cap on the encoded image size in bytes. We refuse to return
25
+ * anything bigger than this even after the model asks for high-res —
26
+ * better to fail loudly than silently nuke the context. */
27
+ const MAX_BYTES = 1_500_000;
28
+
29
+ export const screenshotTool = defineTool({
30
+ name: 'computer_screenshot',
31
+ description:
32
+ 'Take a screenshot of the macOS desktop. Returns base64-encoded image bytes ' +
33
+ 'the model can see directly. By default the image is downscaled to 1280px ' +
34
+ 'and JPEG-compressed so a single screenshot stays well under context — ' +
35
+ 'override `maxDim` / `format` / `quality` only when you genuinely need full ' +
36
+ 'resolution. macOS will prompt for Screen Recording permission on first use.',
37
+ inputSchema: z.object({
38
+ region: regionSchema
39
+ .optional()
40
+ .describe(
41
+ 'Crop to a rectangle in screen-pixel coordinates (top-left origin). Omit for full screen.',
42
+ ),
43
+ maxDim: z
44
+ .number()
45
+ .int()
46
+ .min(256)
47
+ .max(3840)
48
+ .optional()
49
+ .describe(
50
+ `Resize so the longest edge is <= this many pixels. Default ${DEFAULT_MAX_DIM}. ` +
51
+ 'Lower = smaller payload + less context cost; raise only when you need pixel detail.',
52
+ ),
53
+ format: z
54
+ .enum(['jpeg', 'png'])
55
+ .optional()
56
+ .describe(
57
+ `Output format. Default "${DEFAULT_FORMAT}" (much smaller). Pick "png" only when ` +
58
+ 'you need lossless (rare — JPEG is fine for UI screenshots).',
59
+ ),
60
+ quality: z
61
+ .number()
62
+ .int()
63
+ .min(40)
64
+ .max(100)
65
+ .optional()
66
+ .describe(
67
+ `JPEG quality (1-100). Default ${DEFAULT_JPEG_QUALITY}. Ignored for PNG output.`,
68
+ ),
69
+ }),
70
+ permission: { action: 'prompt' },
71
+ async handler({ region, maxDim, format, quality }, ctx) {
72
+ ensureDarwin('computer_screenshot');
73
+ const fmt = format ?? DEFAULT_FORMAT;
74
+ const dim = maxDim ?? DEFAULT_MAX_DIM;
75
+ const q = quality ?? DEFAULT_JPEG_QUALITY;
76
+
77
+ // Always capture as PNG first (preserves color depth + no double
78
+ // compression), then let `sips` resize and convert in one pass.
79
+ // `-x` silences the camera-shutter sound.
80
+ // A random suffix guarantees uniqueness even for two captures in the
81
+ // same process within the same millisecond (pid+Date.now() alone can
82
+ // collide and cross-corrupt the temp files).
83
+ const uniq = randomUUID();
84
+ const captureTmp = path.join(
85
+ os.tmpdir(),
86
+ `moxxy-screencap-${process.pid}-${Date.now()}-${uniq}.png`,
87
+ );
88
+ const captureArgs = ['-x', '-t', 'png'];
89
+ if (region) {
90
+ captureArgs.push('-R', `${region.x},${region.y},${region.width},${region.height}`);
91
+ }
92
+ captureArgs.push(captureTmp);
93
+ // Guarantee the original PNG (potentially several MB for a Retina full
94
+ // screen) is removed on EVERY exit path — success, throw, or a spawn
95
+ // reject (e.g. `sips` not on PATH) / mid-capture timeout that may have
96
+ // left a partial file. Without this, those failure paths leak the temp
97
+ // file in os.tmpdir() permanently and accumulate over repeated failures.
98
+ try {
99
+ const cap = await runProcess('screencapture', captureArgs, {
100
+ ...(ctx.signal ? { signal: ctx.signal } : {}),
101
+ timeoutMs: 15_000,
102
+ });
103
+ if (cap.exitCode !== 0) {
104
+ const cause = procFailureCause(cap, 15_000);
105
+ throw new MoxxyError({
106
+ code: 'TOOL_ERROR',
107
+ message: cause
108
+ ? `screencapture ${cause}`
109
+ : `screencapture failed (exit ${cap.exitCode}): ${cap.stderr.trim() || '(no stderr — likely Screen Recording permission missing — grant in System Settings → Privacy & Security)'}`,
110
+ context: { tool: 'computer_screenshot', exitCode: cap.exitCode, timedOut: cap.timedOut ? 1 : 0 },
111
+ });
112
+ }
113
+
114
+ // Resize + format-convert in one sips call. `-Z N` fits within N
115
+ // on the longest edge while preserving aspect ratio. Output ext
116
+ // picks the format; format options apply when JPEG.
117
+ const outExt = fmt === 'jpeg' ? 'jpg' : 'png';
118
+ const outTmp = path.join(
119
+ os.tmpdir(),
120
+ `moxxy-screencap-${process.pid}-${Date.now()}-${uniq}-out.${outExt}`,
121
+ );
122
+ const sipsArgs = [
123
+ '-Z',
124
+ String(dim),
125
+ '--setProperty',
126
+ 'format',
127
+ fmt,
128
+ ];
129
+ if (fmt === 'jpeg') {
130
+ sipsArgs.push('--setProperty', 'formatOptions', String(q));
131
+ }
132
+ sipsArgs.push(captureTmp, '--out', outTmp);
133
+ const sip = await runProcess('sips', sipsArgs, {
134
+ ...(ctx.signal ? { signal: ctx.signal } : {}),
135
+ timeoutMs: 15_000,
136
+ });
137
+ if (sip.exitCode !== 0) {
138
+ await fs.rm(outTmp, { force: true });
139
+ const cause = procFailureCause(sip, 15_000);
140
+ throw new MoxxyError({
141
+ code: 'TOOL_ERROR',
142
+ message: cause
143
+ ? `sips resize/convert ${cause}`
144
+ : `sips resize/convert failed (exit ${sip.exitCode}): ${sip.stderr.trim() || '(no error message)'}`,
145
+ context: { tool: 'computer_screenshot', exitCode: sip.exitCode, timedOut: sip.timedOut ? 1 : 0 },
146
+ });
147
+ }
148
+
149
+ try {
150
+ const bytes = await fs.readFile(outTmp);
151
+ if (bytes.length > MAX_BYTES) {
152
+ throw new MoxxyError({
153
+ code: 'TOOL_ERROR',
154
+ message:
155
+ `screenshot exceeded ${MAX_BYTES} bytes after compression (got ${bytes.length}). ` +
156
+ `Lower maxDim (currently ${dim}) or quality (currently ${q}), or pass a smaller region.`,
157
+ context: { tool: 'computer_screenshot', byteLength: bytes.length },
158
+ });
159
+ }
160
+ // The `{ mediaType, base64 }` pair is load-bearing, not decorative: the
161
+ // SDK's tool_result projection keys off exactly this shape to emit a
162
+ // provider `image` ContentBlock so the model SEES the pixels. Returning
163
+ // the bytes inside a stringified blob (the JSON.stringify fallback path)
164
+ // would reach the model as base64 TEXT it cannot decode. Extra fields
165
+ // are diagnostic only and ignored by the image projection.
166
+ return {
167
+ mediaType: fmt === 'jpeg' ? ('image/jpeg' as const) : ('image/png' as const),
168
+ base64: bytes.toString('base64'),
169
+ byteLength: bytes.length,
170
+ maxDim: dim,
171
+ format: fmt,
172
+ ...(fmt === 'jpeg' ? { quality: q } : {}),
173
+ };
174
+ } finally {
175
+ await fs.rm(outTmp, { force: true });
176
+ }
177
+ } finally {
178
+ await fs.rm(captureTmp, { force: true });
179
+ }
180
+ },
181
+ });
@@ -0,0 +1,60 @@
1
+ import { defineTool, MoxxyError, z } from '@moxxy/sdk';
2
+ import { ensureDarwin, procFailureCause, runProcess } from '../shell.js';
3
+
4
+ export const typeTool = defineTool({
5
+ name: 'computer_type',
6
+ description:
7
+ 'Type a string into whatever has keyboard focus. Use AFTER clicking the ' +
8
+ 'target field. Macros (cmd+C, etc.) belong in computer_key, not here — this ' +
9
+ 'tool sends each character literally. Requires Accessibility permission.',
10
+ inputSchema: z.object({
11
+ text: z
12
+ .string()
13
+ .max(4000)
14
+ .describe(
15
+ 'The literal text to type. Newlines and tabs are typed as-is. ' +
16
+ 'Pre-existing focus is the target — click first if needed.',
17
+ ),
18
+ }),
19
+ permission: { action: 'prompt' },
20
+ async handler({ text }, ctx) {
21
+ ensureDarwin('computer_type');
22
+ if (text.length === 0) return { ok: true, length: 0 };
23
+ // Serialize the text to an AppleScript string literal ourselves and
24
+ // pass it inline via `osascript -e`, so we never have to escape
25
+ // AppleScript quote rules at the shell layer.
26
+ const literal = toAppleScriptString(text);
27
+ const directScript = `tell application "System Events" to keystroke ${literal}`;
28
+ const proc = await runProcess('osascript', ['-e', directScript], {
29
+ ...(ctx.signal ? { signal: ctx.signal } : {}),
30
+ timeoutMs: 30_000,
31
+ });
32
+ if (proc.exitCode !== 0) {
33
+ const cause = procFailureCause(proc, 30_000);
34
+ throw new MoxxyError({
35
+ code: 'TOOL_ERROR',
36
+ message: cause
37
+ ? `type ${cause}`
38
+ : `type failed (exit ${proc.exitCode}): ${proc.stderr.trim() || '(check Accessibility permission)'}`,
39
+ context: { tool: 'computer_type', exitCode: proc.exitCode, timedOut: proc.timedOut ? 1 : 0 },
40
+ });
41
+ }
42
+ return { ok: true, length: text.length };
43
+ },
44
+ });
45
+
46
+ /**
47
+ * Serialize an arbitrary JS string into a valid AppleScript string
48
+ * literal. AppleScript string syntax: `"..."` with `\"` and `\\` as
49
+ * the only escapes; newlines are written as `" & return & "`.
50
+ */
51
+ export function toAppleScriptString(s: string): string {
52
+ // Split on newlines to use `return` (AppleScript's CR constant) so
53
+ // a multi-line type call sends actual Enter keystrokes rather than a
54
+ // literal `\n` in one keystroke (which keystroke would refuse).
55
+ const parts = s.split('\n').map((line) => {
56
+ const escaped = line.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
57
+ return `"${escaped}"`;
58
+ });
59
+ return parts.join(' & return & ');
60
+ }