@moxxy/plugin-computer-control 0.41.2 → 0.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (269) hide show
  1. package/bin/win32-x64/moxxy-computer.exe +0 -0
  2. package/bin/win32-x64/moxxy-computer.exe.json +1 -1
  3. package/dist/backend/access.d.ts +129 -0
  4. package/dist/backend/access.d.ts.map +1 -0
  5. package/dist/backend/access.js +158 -0
  6. package/dist/backend/access.js.map +1 -0
  7. package/dist/backend/app-hints.d.ts +14 -0
  8. package/dist/backend/app-hints.d.ts.map +1 -0
  9. package/dist/backend/app-hints.js +30 -0
  10. package/dist/backend/app-hints.js.map +1 -0
  11. package/dist/backend/backend.d.ts +70 -0
  12. package/dist/backend/backend.d.ts.map +1 -0
  13. package/dist/backend/backend.js +466 -0
  14. package/dist/backend/backend.js.map +1 -0
  15. package/dist/backend/rpc.d.ts +1438 -0
  16. package/dist/backend/rpc.d.ts.map +1 -0
  17. package/dist/backend/rpc.js +75 -0
  18. package/dist/backend/rpc.js.map +1 -0
  19. package/dist/backend/turn-controls.d.ts +20 -0
  20. package/dist/backend/turn-controls.d.ts.map +1 -0
  21. package/dist/backend/turn-controls.js +115 -0
  22. package/dist/backend/turn-controls.js.map +1 -0
  23. package/dist/contract/guidance.d.ts +12 -0
  24. package/dist/contract/guidance.d.ts.map +1 -0
  25. package/dist/contract/guidance.js +42 -0
  26. package/dist/contract/guidance.js.map +1 -0
  27. package/dist/contract/image.d.ts +29 -0
  28. package/dist/contract/image.d.ts.map +1 -0
  29. package/dist/contract/image.js +46 -0
  30. package/dist/contract/image.js.map +1 -0
  31. package/dist/contract/keys.d.ts +15 -0
  32. package/dist/contract/keys.d.ts.map +1 -0
  33. package/dist/contract/keys.js +118 -0
  34. package/dist/contract/keys.js.map +1 -0
  35. package/dist/contract/outcome.d.ts +61 -0
  36. package/dist/contract/outcome.d.ts.map +1 -0
  37. package/dist/contract/outcome.js +63 -0
  38. package/dist/contract/outcome.js.map +1 -0
  39. package/dist/contract/progress.d.ts +27 -0
  40. package/dist/contract/progress.d.ts.map +1 -0
  41. package/dist/contract/progress.js +36 -0
  42. package/dist/contract/progress.js.map +1 -0
  43. package/dist/contract/tools.d.ts +338 -0
  44. package/dist/contract/tools.d.ts.map +1 -0
  45. package/dist/contract/tools.js +195 -0
  46. package/dist/contract/tools.js.map +1 -0
  47. package/dist/contract/untrusted.d.ts +3 -0
  48. package/dist/contract/untrusted.d.ts.map +1 -0
  49. package/dist/contract/untrusted.js +8 -0
  50. package/dist/contract/untrusted.js.map +1 -0
  51. package/dist/helper/artifact.d.ts +41 -0
  52. package/dist/helper/artifact.d.ts.map +1 -0
  53. package/dist/helper/artifact.js +189 -0
  54. package/dist/helper/artifact.js.map +1 -0
  55. package/dist/helper/protocol.d.ts +80 -0
  56. package/dist/helper/protocol.d.ts.map +1 -0
  57. package/dist/helper/protocol.js +49 -0
  58. package/dist/helper/protocol.js.map +1 -0
  59. package/dist/helper/transport.d.ts +43 -0
  60. package/dist/helper/transport.d.ts.map +1 -0
  61. package/dist/{windows → helper}/transport.js +45 -19
  62. package/dist/helper/transport.js.map +1 -0
  63. package/dist/index.d.ts +9 -22
  64. package/dist/index.d.ts.map +1 -1
  65. package/dist/index.js +35 -42
  66. package/dist/index.js.map +1 -1
  67. package/dist/jev/ladder.d.ts +32 -0
  68. package/dist/jev/ladder.d.ts.map +1 -0
  69. package/dist/jev/ladder.js +66 -0
  70. package/dist/jev/ladder.js.map +1 -0
  71. package/dist/jev/memory.d.ts +11 -0
  72. package/dist/jev/memory.d.ts.map +1 -0
  73. package/dist/jev/memory.js +16 -0
  74. package/dist/jev/memory.js.map +1 -0
  75. package/dist/jev/run.d.ts +93 -0
  76. package/dist/jev/run.d.ts.map +1 -0
  77. package/dist/jev/run.js +403 -0
  78. package/dist/jev/run.js.map +1 -0
  79. package/dist/jev/train.d.ts +97 -0
  80. package/dist/jev/train.d.ts.map +1 -0
  81. package/dist/jev/train.js +28 -0
  82. package/dist/jev/train.js.map +1 -0
  83. package/dist/linux/profile.d.ts +6 -0
  84. package/dist/linux/profile.d.ts.map +1 -0
  85. package/dist/linux/profile.js +21 -0
  86. package/dist/linux/profile.js.map +1 -0
  87. package/dist/macos/profile.d.ts +5 -0
  88. package/dist/macos/profile.d.ts.map +1 -0
  89. package/dist/macos/profile.js +17 -0
  90. package/dist/macos/profile.js.map +1 -0
  91. package/dist/preview/controller.d.ts +127 -0
  92. package/dist/preview/controller.d.ts.map +1 -0
  93. package/dist/preview/controller.js +203 -0
  94. package/dist/preview/controller.js.map +1 -0
  95. package/dist/preview/surface.d.ts +9 -0
  96. package/dist/preview/surface.d.ts.map +1 -0
  97. package/dist/preview/surface.js +50 -0
  98. package/dist/preview/surface.js.map +1 -0
  99. package/dist/windows/maintenance.d.ts +1 -1
  100. package/dist/windows/maintenance.d.ts.map +1 -1
  101. package/dist/windows/maintenance.js +6 -5
  102. package/dist/windows/maintenance.js.map +1 -1
  103. package/dist/windows/profile.d.ts +5 -0
  104. package/dist/windows/profile.d.ts.map +1 -0
  105. package/dist/windows/profile.js +17 -0
  106. package/dist/windows/profile.js.map +1 -0
  107. package/learned/README.md +26 -0
  108. package/learned/cases/com.apple.calculator.json +395 -0
  109. package/learned/cases/com.apple.finder.json +115 -0
  110. package/learned/cases/com.apple.safari.json +112 -0
  111. package/learned/cases/com.apple.systempreferences.json +255 -0
  112. package/learned/com.apple.calculator-eace95fc.json +334 -0
  113. package/learned/com.apple.finder-27cf6ce8.json +260 -0
  114. package/learned/com.apple.safari-7cd9df4f.json +143 -0
  115. package/learned/com.apple.systempreferences-02cf0b8b.json +535 -0
  116. package/package.json +13 -7
  117. package/scripts/promote-learned.mjs +8 -0
  118. package/scripts/summarize-trial.mjs +68 -0
  119. package/scripts/train-learned.mjs +64 -0
  120. package/skills/computer-apps/blender.md +29 -0
  121. package/skills/computer-apps/browsers.md +50 -0
  122. package/skills/computer-apps/design-tools.md +37 -0
  123. package/skills/computer-apps/finder.md +23 -0
  124. package/skills/computer-apps/office.md +46 -0
  125. package/skills/computer-apps/video-editors.md +44 -0
  126. package/skills/computer-control.md +154 -191
  127. package/src/backend/access.test.ts +158 -0
  128. package/src/backend/access.ts +168 -0
  129. package/src/backend/app-hints.test.ts +59 -0
  130. package/src/backend/app-hints.ts +39 -0
  131. package/src/backend/backend.test.ts +725 -0
  132. package/src/backend/backend.ts +482 -0
  133. package/src/backend/contract-helper.fixture.mjs +138 -0
  134. package/src/backend/helper.fixture.ts +38 -0
  135. package/src/backend/rpc.ts +90 -0
  136. package/src/backend/turn-controls.test.ts +178 -0
  137. package/src/backend/turn-controls.ts +120 -0
  138. package/src/contract/guidance.test.ts +75 -0
  139. package/src/contract/guidance.ts +48 -0
  140. package/src/contract/image.test.ts +74 -0
  141. package/src/contract/image.ts +54 -0
  142. package/src/contract/keys.test.ts +103 -0
  143. package/src/contract/keys.ts +117 -0
  144. package/src/contract/outcome.test.ts +71 -0
  145. package/src/contract/outcome.ts +72 -0
  146. package/src/contract/progress.test.ts +56 -0
  147. package/src/contract/progress.ts +43 -0
  148. package/src/contract/tools.test.ts +210 -0
  149. package/src/contract/tools.ts +212 -0
  150. package/src/contract/untrusted.test.ts +20 -0
  151. package/src/contract/untrusted.ts +8 -0
  152. package/src/helper/artifact.test.ts +131 -0
  153. package/src/helper/artifact.ts +182 -0
  154. package/src/helper/protocol.test.ts +29 -0
  155. package/src/helper/protocol.ts +50 -0
  156. package/src/{windows → helper}/transport.test.ts +76 -19
  157. package/src/{windows → helper}/transport.ts +54 -16
  158. package/src/index.test.ts +112 -0
  159. package/src/index.ts +36 -53
  160. package/src/jev/ladder.test.ts +113 -0
  161. package/src/jev/ladder.ts +88 -0
  162. package/src/jev/memory.ts +20 -0
  163. package/src/jev/run.test.ts +700 -0
  164. package/src/jev/run.ts +443 -0
  165. package/src/jev/train.test.ts +40 -0
  166. package/src/jev/train.ts +41 -0
  167. package/src/linux/helper.test.ts +481 -0
  168. package/src/linux/profile.ts +25 -0
  169. package/src/macos/helper.test.ts +988 -0
  170. package/src/macos/profile.ts +19 -0
  171. package/src/preview/controller.test.ts +308 -0
  172. package/src/preview/controller.ts +262 -0
  173. package/src/preview/surface.test.ts +68 -0
  174. package/src/preview/surface.ts +45 -0
  175. package/src/skill.test.ts +34 -0
  176. package/src/windows/maintenance.ts +8 -7
  177. package/src/windows/profile.ts +19 -0
  178. package/dist/shell.d.ts +0 -56
  179. package/dist/shell.d.ts.map +0 -1
  180. package/dist/shell.js +0 -189
  181. package/dist/shell.js.map +0 -1
  182. package/dist/temporary-files.d.ts +0 -2
  183. package/dist/temporary-files.d.ts.map +0 -1
  184. package/dist/temporary-files.js +0 -10
  185. package/dist/temporary-files.js.map +0 -1
  186. package/dist/tools/applescript.d.ts +0 -2
  187. package/dist/tools/applescript.d.ts.map +0 -1
  188. package/dist/tools/applescript.js +0 -50
  189. package/dist/tools/applescript.js.map +0 -1
  190. package/dist/tools/click.d.ts +0 -2
  191. package/dist/tools/click.d.ts.map +0 -1
  192. package/dist/tools/click.js +0 -58
  193. package/dist/tools/click.js.map +0 -1
  194. package/dist/tools/clipboard.d.ts +0 -2
  195. package/dist/tools/clipboard.d.ts.map +0 -1
  196. package/dist/tools/clipboard.js +0 -71
  197. package/dist/tools/clipboard.js.map +0 -1
  198. package/dist/tools/key.d.ts +0 -11
  199. package/dist/tools/key.d.ts.map +0 -1
  200. package/dist/tools/key.js +0 -143
  201. package/dist/tools/key.js.map +0 -1
  202. package/dist/tools/open.d.ts +0 -2
  203. package/dist/tools/open.d.ts.map +0 -1
  204. package/dist/tools/open.js +0 -91
  205. package/dist/tools/open.js.map +0 -1
  206. package/dist/tools/screenshot.d.ts +0 -2
  207. package/dist/tools/screenshot.d.ts.map +0 -1
  208. package/dist/tools/screenshot.js +0 -162
  209. package/dist/tools/screenshot.js.map +0 -1
  210. package/dist/tools/type.d.ts +0 -8
  211. package/dist/tools/type.d.ts.map +0 -1
  212. package/dist/tools/type.js +0 -65
  213. package/dist/tools/type.js.map +0 -1
  214. package/dist/windows/artifact.d.ts +0 -3
  215. package/dist/windows/artifact.d.ts.map +0 -1
  216. package/dist/windows/artifact.js +0 -57
  217. package/dist/windows/artifact.js.map +0 -1
  218. package/dist/windows/backend.d.ts +0 -11
  219. package/dist/windows/backend.d.ts.map +0 -1
  220. package/dist/windows/backend.js +0 -122
  221. package/dist/windows/backend.js.map +0 -1
  222. package/dist/windows/contracts.d.ts +0 -1627
  223. package/dist/windows/contracts.d.ts.map +0 -1
  224. package/dist/windows/contracts.js +0 -137
  225. package/dist/windows/contracts.js.map +0 -1
  226. package/dist/windows/control-service.d.ts +0 -12
  227. package/dist/windows/control-service.d.ts.map +0 -1
  228. package/dist/windows/control-service.js +0 -65
  229. package/dist/windows/control-service.js.map +0 -1
  230. package/dist/windows/guidance.d.ts +0 -3
  231. package/dist/windows/guidance.d.ts.map +0 -1
  232. package/dist/windows/guidance.js +0 -20
  233. package/dist/windows/guidance.js.map +0 -1
  234. package/dist/windows/protocol.d.ts +0 -9
  235. package/dist/windows/protocol.d.ts.map +0 -1
  236. package/dist/windows/protocol.js +0 -32
  237. package/dist/windows/protocol.js.map +0 -1
  238. package/dist/windows/transport.d.ts +0 -23
  239. package/dist/windows/transport.d.ts.map +0 -1
  240. package/dist/windows/transport.js.map +0 -1
  241. package/src/shell.test.ts +0 -186
  242. package/src/shell.ts +0 -213
  243. package/src/temporary-files.test.ts +0 -18
  244. package/src/temporary-files.ts +0 -9
  245. package/src/tools/applescript-serialize.test.ts +0 -74
  246. package/src/tools/applescript.ts +0 -53
  247. package/src/tools/click.ts +0 -60
  248. package/src/tools/clipboard.ts +0 -72
  249. package/src/tools/key.ts +0 -155
  250. package/src/tools/open.ts +0 -96
  251. package/src/tools/screenshot.test.ts +0 -137
  252. package/src/tools/screenshot.ts +0 -180
  253. package/src/tools/type.ts +0 -68
  254. package/src/tools.test.ts +0 -94
  255. package/src/windows/action-contracts.test.ts +0 -13
  256. package/src/windows/artifact.test.ts +0 -16
  257. package/src/windows/artifact.ts +0 -57
  258. package/src/windows/backend.test.ts +0 -41
  259. package/src/windows/backend.ts +0 -122
  260. package/src/windows/contracts.test.ts +0 -81
  261. package/src/windows/contracts.ts +0 -143
  262. package/src/windows/control-service.test.ts +0 -58
  263. package/src/windows/control-service.ts +0 -68
  264. package/src/windows/guidance.test.ts +0 -29
  265. package/src/windows/guidance.ts +0 -21
  266. package/src/windows/model-contract.test.ts +0 -37
  267. package/src/windows/protocol.ts +0 -27
  268. package/src/windows/text-contracts.test.ts +0 -14
  269. package/src/windows/window-typing.test.ts +0 -14
@@ -0,0 +1,182 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { constants, existsSync, readFileSync } from 'node:fs';
3
+ import { lstat, open, rename, writeFile } from 'node:fs/promises';
4
+ import type { FileHandle } from 'node:fs/promises';
5
+ import { z } from 'zod';
6
+
7
+ const manifestSchema = (protocolVersion: number) => z.object({
8
+ protocolVersion: z.literal(protocolVersion),
9
+ /** Absent for Windows and macOS, which the architecture alone tells apart. */
10
+ os: z.literal('linux').optional(),
11
+ architecture: z.enum(['x64', 'arm64', 'x86_64', 'universal']),
12
+ sha256: z.string().regex(/^[a-f0-9]{64}$/),
13
+ }).strict();
14
+ type Architecture = z.infer<ReturnType<typeof manifestSchema>>['architecture'];
15
+
16
+ const MACH_O_64 = 0xfeedfacf;
17
+ const MACH_O_FAT = 0xcafebabe;
18
+ const CPU = { arm64: 0x0100000c, x86_64: 0x01000007 } as const;
19
+
20
+ function assertWindowsX64(bytes: Buffer): void {
21
+ if (bytes.length < 64 || bytes.toString('ascii', 0, 2) !== 'MZ') throw new Error('Invalid Computer Use executable');
22
+ const offset = bytes.readUInt32LE(60);
23
+ if (offset > bytes.length - 6 || bytes.toString('ascii', offset, offset + 4) !== 'PE\0\0' || bytes.readUInt16LE(offset + 4) !== 0x8664) {
24
+ throw new Error('Computer Use requires a Windows x64 executable');
25
+ }
26
+ }
27
+
28
+ const ELF_MACHINE: Partial<Record<Architecture, number>> = { x64: 62, arm64: 183 };
29
+
30
+ function assertLinux(bytes: Buffer, architecture: Architecture): void {
31
+ const machine = ELF_MACHINE[architecture];
32
+ // 64-bit, little-endian ELF for the named machine.
33
+ const matches = machine !== undefined && bytes.length >= 20 && bytes.toString('latin1', 0, 4) === '\x7fELF'
34
+ && bytes[4] === 2 && bytes[5] === 1 && bytes.readUInt16LE(18) === machine;
35
+ if (!matches) throw new Error(`Computer Use requires a Linux ${architecture} executable`);
36
+ }
37
+
38
+ function assertMacOS(bytes: Buffer, architecture: Exclude<Architecture, 'x64'>): void {
39
+ const cpus = (machOSlices(bytes) ?? []).map(({ cpu }) => cpu);
40
+ const required = architecture === 'universal' ? [CPU.arm64, CPU.x86_64] : [CPU[architecture]];
41
+ if (!required.every((cpu) => cpus.includes(cpu))) throw new Error(`Computer Use requires a macOS ${architecture} executable`);
42
+ }
43
+
44
+ const LC_SEGMENT_64 = 0x19;
45
+ const LC_CODE_SIGNATURE = 0x1d;
46
+ const invalid = () => new Error('Invalid Computer Use executable');
47
+
48
+ /** One Mach-O slice without its code signature, with the fields signing rewrites set to zero. */
49
+ function unsignedSlice(slice: Buffer): Buffer {
50
+ if (slice.length < 32 || slice.readUInt32LE(0) !== MACH_O_64) throw invalid();
51
+ const copy = Buffer.from(slice);
52
+ const commands = copy.readUInt32LE(16);
53
+ if (32 + copy.readUInt32LE(20) > copy.length) throw invalid();
54
+ let end = copy.length;
55
+ let offset = 32;
56
+ for (let index = 0; index < commands; index++) {
57
+ if (offset + 8 > copy.length) throw invalid();
58
+ const command = copy.readUInt32LE(offset);
59
+ const size = copy.readUInt32LE(offset + 4);
60
+ if (size < 8 || offset + size > copy.length) throw invalid();
61
+ if (command === LC_CODE_SIGNATURE && size >= 16) {
62
+ end = Math.min(end, copy.readUInt32LE(offset + 8));
63
+ copy.fill(0, offset + 8, offset + 16);
64
+ } else if (command === LC_SEGMENT_64 && size >= 72 && copy.toString('latin1', offset + 8, offset + 24).replace(/\0+$/, '') === '__LINKEDIT') {
65
+ // The signature is the tail of __LINKEDIT, so its sizes move with it.
66
+ copy.fill(0, offset + 32, offset + 40);
67
+ copy.fill(0, offset + 48, offset + 56);
68
+ }
69
+ offset += size;
70
+ }
71
+ return copy.subarray(0, end);
72
+ }
73
+
74
+ /** The slices of a thin or fat Mach-O with their CPU type; `undefined` when the bytes are not Mach-O. */
75
+ function machOSlices(bytes: Buffer): Array<{ cpu: number; slice: Buffer }> | undefined {
76
+ if (bytes.length < 8) return undefined;
77
+ if (bytes.readUInt32LE(0) === MACH_O_64) return [{ cpu: bytes.readUInt32LE(4), slice: bytes }];
78
+ if (bytes.readUInt32BE(0) !== MACH_O_FAT) return undefined;
79
+ const count = bytes.readUInt32BE(4);
80
+ if (count > 16 || 8 + count * 20 > bytes.length) throw invalid();
81
+ return Array.from({ length: count }, (_, index) => {
82
+ const entry = 8 + index * 20;
83
+ const start = bytes.readUInt32BE(entry + 8);
84
+ const size = bytes.readUInt32BE(entry + 12);
85
+ if (start + size > bytes.length) throw invalid();
86
+ return { cpu: bytes.readUInt32BE(entry), slice: bytes.subarray(start, start + size) };
87
+ });
88
+ }
89
+
90
+ /**
91
+ * The digest a manifest records. A Windows or Linux executable is hashed whole. A Mach-O
92
+ * is hashed without its code signatures, because packaging the desktop app signs
93
+ * every executable again after the manifest was written; macOS itself refuses to
94
+ * run code whose signature does not match.
95
+ */
96
+ export function helperDigest(bytes: Buffer): string {
97
+ const hash = createHash('sha256');
98
+ const slices = machOSlices(bytes);
99
+ if (!slices) return hash.update(bytes).digest('hex');
100
+ for (const { cpu, slice } of slices) {
101
+ const header = Buffer.alloc(4);
102
+ header.writeUInt32BE(cpu);
103
+ // An empty fat entry (synthetic headers in tests) has no code to hash.
104
+ hash.update(header).update(slice.length === 0 ? slice : unsignedSlice(slice));
105
+ }
106
+ return hash.digest('hex');
107
+ }
108
+
109
+ export function validateHelperArtifact(bytes: Buffer, manifest: unknown, protocolVersion: number): void {
110
+ const expected = manifestSchema(protocolVersion).parse(manifest);
111
+ if (expected.os === 'linux') assertLinux(bytes, expected.architecture);
112
+ else if (expected.architecture === 'x64') assertWindowsX64(bytes);
113
+ else assertMacOS(bytes, expected.architecture);
114
+ if (helperDigest(bytes) !== expected.sha256) throw new Error('Computer Use executable checksum mismatch');
115
+ }
116
+
117
+ // POSIX-only; Node leaves it undefined on Windows.
118
+ const O_NOFOLLOW = constants.O_NOFOLLOW ?? 0;
119
+
120
+ /**
121
+ * Read a regular file no larger than `limit`, checking the open handle rather
122
+ * than the path so the bytes that are validated are the bytes that were
123
+ * measured. A `stat` followed by `readFile` can be repointed in between, which
124
+ * would let an oversized file through the size guard.
125
+ */
126
+ async function readBounded(file: string, limit: number): Promise<Buffer> {
127
+ if (O_NOFOLLOW === 0 && (await lstat(file)).isSymbolicLink()) {
128
+ throw new Error('Computer Use artifact exceeds size limit');
129
+ }
130
+ let handle: FileHandle | undefined;
131
+ try {
132
+ handle = await open(file, constants.O_RDONLY | O_NOFOLLOW);
133
+ const info = await handle.stat();
134
+ if (!info.isFile() || info.size > limit) throw new Error('Computer Use artifact exceeds size limit');
135
+ return await handle.readFile();
136
+ } catch (error) {
137
+ // O_NOFOLLOW reports a symlink as ELOOP.
138
+ if ((error as NodeJS.ErrnoException).code === 'ELOOP') {
139
+ throw new Error('Computer Use artifact exceeds size limit');
140
+ }
141
+ throw error;
142
+ } finally {
143
+ await handle?.close();
144
+ }
145
+ }
146
+
147
+ export async function verifyHelperArtifact(executable: string, protocolVersion: number): Promise<void> {
148
+ const manifestPath = executable + '.json';
149
+ const [bytes, manifest] = await Promise.all([
150
+ readBounded(executable, 32_000_000),
151
+ readBounded(manifestPath, 4096),
152
+ ]);
153
+ validateHelperArtifact(bytes, JSON.parse(manifest.toString('utf8')), protocolVersion);
154
+ }
155
+
156
+ /** Written by the native build next to the executable it just produced. */
157
+ export async function writeHelperManifest(executable: string, manifest: { protocolVersion: number; architecture: Architecture; os?: 'linux' }): Promise<void> {
158
+ const sha256 = helperDigest(await readBounded(executable, 32_000_000));
159
+ const temporary = `${executable}.json.tmp`;
160
+ await writeFile(temporary, JSON.stringify({ ...manifest, sha256 }));
161
+ await rename(temporary, `${executable}.json`);
162
+ }
163
+
164
+ /**
165
+ * Why a helper cannot be used, known without reading the executable: decides
166
+ * which tools a plugin offers. The full check runs before every launch.
167
+ */
168
+ export function helperProblem(executable: string, protocolVersion: number): string | undefined {
169
+ if (!existsSync(executable)) return 'The Computer Use helper is missing.';
170
+ let manifest: unknown;
171
+ try {
172
+ manifest = JSON.parse(readFileSync(`${executable}.json`, 'utf8'));
173
+ } catch {
174
+ return 'The Computer Use helper has no readable manifest.';
175
+ }
176
+ const found = z.object({ protocolVersion: z.number().int() }).safeParse(manifest);
177
+ if (!found.success) return 'The Computer Use helper has no readable manifest.';
178
+ if (found.data.protocolVersion !== protocolVersion) {
179
+ return `The Computer Use helper speaks protocol ${found.data.protocolVersion}; this version needs ${protocolVersion}.`;
180
+ }
181
+ return undefined;
182
+ }
@@ -0,0 +1,29 @@
1
+ import { describe, expect, it } from 'vitest';
2
+ import { JsonLineDecoder, responseSchemaFor } from './protocol.js';
3
+
4
+ describe('bounded JSON lines', () => {
5
+ it('decodes split UTF-8 and multiple frames without losing bytes', () => {
6
+ const decoder = new JsonLineDecoder(128);
7
+ const bytes = Buffer.from('{"text":"żółć"}\n{}\n');
8
+ const result: unknown[] = [];
9
+ for (const byte of bytes) result.push(...decoder.push(Buffer.from([byte])));
10
+ expect(result).toEqual([{ text: 'żółć' }, {}]);
11
+ });
12
+ it('rejects oversize, malformed and incomplete messages', () => {
13
+ expect(() => new JsonLineDecoder(3).push(Buffer.from('1234'))).toThrow(/limit/);
14
+ expect(() => new JsonLineDecoder(100).push(Buffer.from('{bad}\n'))).toThrow();
15
+ const decoder = new JsonLineDecoder(100);
16
+ decoder.push(Buffer.from('{'));
17
+ expect(() => decoder.finish()).toThrow(/incomplete/);
18
+ });
19
+ });
20
+
21
+ describe('response envelope', () => {
22
+ it('accepts only the configured protocol version and a well-formed result or error', () => {
23
+ const response = responseSchemaFor(5);
24
+ expect(response.safeParse({ version: 5, id: '1', ok: true, result: {} }).success).toBe(true);
25
+ expect(response.safeParse({ version: 5, id: '1', ok: false, error: { code: 'x', message: 'y' } }).success).toBe(true);
26
+ expect(response.safeParse({ version: 4, id: '1', ok: true, result: {} }).success).toBe(false);
27
+ expect(response.safeParse({ version: 5, id: '1', ok: false }).success).toBe(false);
28
+ });
29
+ });
@@ -0,0 +1,50 @@
1
+ import { z } from 'zod';
2
+
3
+ /** Per-frame byte ceiling shared by every native helper. */
4
+ export const MAX_FRAME_BYTES = 3_000_000;
5
+ const idSchema = z.string().min(1).max(160);
6
+
7
+ export const controlCommandSchema = z.enum(['pause', 'resume', 'stop', 'takeover']);
8
+ /** The first helper protocol that understands `takeover`; older helpers are paused instead. */
9
+ export const TAKEOVER_PROTOCOL_VERSION = 5;
10
+ export const controlStates = ['idle', 'background', 'foreground', 'waiting_for_focus', 'paused_by_user', 'recovering', 'stopped', 'failed'] as const;
11
+
12
+ /** A request-correlated state change; the transport suspends the request deadline while waiting. */
13
+ export const controlStateSchemaFor = (version: number) => z.object({
14
+ version: z.literal(version), event: z.literal('control_state'), id: idSchema, state: z.enum(controlStates),
15
+ }).strict();
16
+ export type ControlState = z.infer<ReturnType<typeof controlStateSchemaFor>>;
17
+
18
+ export const responseSchemaFor = (version: number) => z.discriminatedUnion('ok', [
19
+ z.object({ version: z.literal(version), id: idSchema, ok: z.literal(true), result: z.unknown() }).strict(),
20
+ z.object({ version: z.literal(version), id: idSchema, ok: z.literal(false),
21
+ error: z.object({ code: z.string().max(80), message: z.string().max(2048) }).strict() }).strict(),
22
+ ]);
23
+
24
+ /** A byte-bounded decoder; UTF-8 is decoded only after a complete frame arrives. */
25
+ export class JsonLineDecoder {
26
+ private pending = Buffer.alloc(0);
27
+ constructor(private readonly limit: number) {}
28
+
29
+ push(bytes: Buffer): unknown[] {
30
+ const frames: unknown[] = [];
31
+ let offset = 0;
32
+ while (offset < bytes.length) {
33
+ const newline = bytes.indexOf(10, offset);
34
+ const end = newline < 0 ? bytes.length : newline;
35
+ const part = bytes.subarray(offset, end);
36
+ if (this.pending.length + part.length > this.limit) throw new Error('Computer Use protocol frame limit exceeded');
37
+ this.pending = Buffer.concat([this.pending, part]);
38
+ if (newline < 0) break;
39
+ const text = new TextDecoder('utf-8', { fatal: true }).decode(this.pending);
40
+ frames.push(JSON.parse(text));
41
+ this.pending = Buffer.alloc(0);
42
+ offset = newline + 1;
43
+ }
44
+ return frames;
45
+ }
46
+
47
+ finish(): void {
48
+ if (this.pending.length) throw new Error('Computer Use protocol incomplete frame');
49
+ }
50
+ }
@@ -1,14 +1,19 @@
1
1
  import { describe, expect, it } from 'vitest';
2
- import { HelperTransport } from './transport.js';
2
+ import { z } from 'zod';
3
+ import { HelperTransport, type HelperEvent, type HelperTransportOptions } from './transport.js';
3
4
 
4
- // Real subprocesses exercise pipe framing/lifetime; these do not simulate Windows APIs.
5
+ // Real subprocesses exercise pipe framing/lifetime; these do not simulate platform APIs.
6
+ const helper = (script: string, options: Partial<HelperTransportOptions> = {}) =>
7
+ new HelperTransport(process.execPath, ['-e', script], { protocolVersion: 4, ...options });
5
8
  const peer = `process.stdin.once('data', bytes => {
6
9
  const request = JSON.parse(bytes.toString());
7
10
  process.stdout.write(JSON.stringify({version:4,id:request.id,ok:true,result:{received:request.method}})+'\\n');
8
11
  });`;
12
+ const cursorEvent = z.object({ version: z.literal(4), event: z.literal('cursor'), x: z.number(), y: z.number() }).strict();
13
+
9
14
  describe('native helper transport', () => {
10
15
  it('reports the native panel Stop exit as cancellation, not a helper crash', async () => {
11
- const transport = new HelperTransport(process.execPath, ['-e', "process.stdin.once('data', () => process.exit(20))"]);
16
+ const transport = helper("process.stdin.once('data', () => process.exit(20))");
12
17
  try {
13
18
  await expect(transport.request('click', {}, new AbortController().signal)).rejects.toThrow('Computer Use stopped by user');
14
19
  expect(transport.stoppedByUser).toBe(true);
@@ -17,21 +22,23 @@ describe('native helper transport', () => {
17
22
  });
18
23
  it('excludes explicit focus waiting from the active request timeout', async () => {
19
24
  const states: string[] = [];
20
- const transport = new HelperTransport(process.execPath, ['-e', `process.stdin.once('data', bytes => {
25
+ // The budget also covers the helper's own start, which on a busy Windows
26
+ // runner outlasts 150 ms; the focus wait still outlasts the whole budget.
27
+ const transport = helper(`process.stdin.once('data', bytes => {
21
28
  const r=JSON.parse(bytes.toString());
22
29
  process.stdout.write(JSON.stringify({version:4,event:'control_state',id:r.id,state:'waiting_for_focus'})+'\\n');
23
30
  setTimeout(()=>{
24
31
  process.stdout.write(JSON.stringify({version:4,event:'control_state',id:r.id,state:'foreground'})+'\\n');
25
32
  process.stdout.write(JSON.stringify({version:4,id:r.id,ok:true,result:{delivered:false,status:'needs_observation'}})+'\\n');
26
- },250);
27
- });`], 150, state => states.push(state.state));
33
+ },1500);
34
+ });`, { timeoutMs: 1000, onEvent: (event) => { if (event.event === 'control_state') states.push(String(event.state)); } });
28
35
  try {
29
36
  expect(await transport.request('click', {}, new AbortController().signal)).toEqual({delivered:false,status:'needs_observation'});
30
37
  expect(states).toEqual(['waiting_for_focus','foreground']);
31
38
  } finally { await transport.close(); }
32
39
  });
33
40
  it('exchanges a correlated frame through real private pipes', async () => {
34
- const transport = new HelperTransport(process.execPath, ['-e', peer]);
41
+ const transport = helper(peer);
35
42
  try {
36
43
  expect(await transport.request('status', {}, new AbortController().signal)).toEqual({ received: 'status' });
37
44
  } finally { await transport.close(); }
@@ -39,46 +46,66 @@ describe('native helper transport', () => {
39
46
  });
40
47
  it('cancels while focus waiting is suspended', async () => {
41
48
  const abort = new AbortController();
42
- const transport = new HelperTransport(process.execPath, ['-e', `process.stdin.once('data', bytes => {
49
+ const transport = helper(`process.stdin.once('data', bytes => {
43
50
  const r=JSON.parse(bytes.toString());
44
51
  process.stdout.write(JSON.stringify({version:4,event:'control_state',id:r.id,state:'waiting_for_focus'})+'\\n');
45
52
  process.stdin.resume();
46
- });`], 1000, () => abort.abort());
53
+ });`, { timeoutMs: 1000, onEvent: () => abort.abort() });
47
54
  await expect(transport.request('key', {}, abort.signal)).rejects.toThrow(/cancelled/);
48
55
  await transport.close();
49
56
  expect(transport.closed).toBe(true);
50
57
  });
51
58
  it('still times out after focus waiting ends without an operation response', async () => {
52
- const transport = new HelperTransport(process.execPath, ['-e', `process.stdin.once('data', bytes => {
59
+ const transport = helper(`process.stdin.once('data', bytes => {
53
60
  const r=JSON.parse(bytes.toString());
54
61
  const state=s=>process.stdout.write(JSON.stringify({version:4,event:'control_state',id:r.id,state:s})+'\\n');
55
62
  state('waiting_for_focus');
56
63
  setTimeout(()=>{state('foreground');process.stdin.resume();},200);
57
- });`], 150);
64
+ });`, { timeoutMs: 150 });
58
65
  await expect(transport.request('key', {}, new AbortController().signal)).rejects.toThrow(/timed out/);
59
66
  await transport.close();
60
67
  });
61
68
  it('rejects a control event for a different request', async () => {
62
- const transport = new HelperTransport(process.execPath, ['-e', `process.stdin.once('data', () => {
69
+ const transport = helper(`process.stdin.once('data', () => {
63
70
  process.stdout.write(JSON.stringify({version:4,event:'control_state',id:'wrong',state:'waiting_for_focus'})+'\\n');
64
- });`]);
71
+ });`);
65
72
  await expect(transport.request('key', {}, new AbortController().signal)).rejects.toThrow(/protocol/);
66
73
  await transport.close();
67
74
  });
75
+ it('keeps the helper error code on a refused request', async () => {
76
+ const transport = helper(`process.stdin.once('data', bytes => {
77
+ const request = JSON.parse(bytes.toString());
78
+ process.stdout.write(JSON.stringify({version:4,id:request.id,ok:false,error:{code:'tier_insufficient',message:'read only'}})+'\\n');
79
+ process.stdin.resume();
80
+ });`);
81
+ await expect(transport.request('act', {}, new AbortController().signal))
82
+ .rejects.toMatchObject({ name: 'HelperError', code: 'tier_insufficient', message: 'tier_insufficient: read only' });
83
+ expect(transport.closed).toBe(false);
84
+ await transport.close();
85
+ });
68
86
  it('rejects a mismatched protocol and permanently retires the peer', async () => {
69
- const transport = new HelperTransport(process.execPath, ['-e', peer.replace('version:4', 'version:2')]);
87
+ const transport = helper(peer.replace('version:4', 'version:2'));
70
88
  await expect(transport.request('status', {}, new AbortController().signal)).rejects.toThrow(/protocol/i);
71
89
  await transport.close();
72
90
  expect(transport.closed).toBe(true);
73
91
  });
92
+ it('speaks the protocol version it was configured with', async () => {
93
+ const transport = new HelperTransport(process.execPath, ['-e', `process.stdin.once('data', bytes => {
94
+ const r=JSON.parse(bytes.toString());
95
+ process.stdout.write(JSON.stringify({version:5,id:r.id,ok:true,result:{sent:r.version}})+'\\n');
96
+ });`], { protocolVersion: 5 });
97
+ try {
98
+ expect(await transport.request('status', {}, new AbortController().signal)).toEqual({ sent: 5 });
99
+ } finally { await transport.close(); }
100
+ });
74
101
  it('times out without retrying an ambiguous input operation', async () => {
75
- const transport = new HelperTransport(process.execPath, ['-e', 'process.stdin.resume()'], 40);
102
+ const transport = helper('process.stdin.resume()', { timeoutMs: 40 });
76
103
  await expect(transport.request('type', {}, new AbortController().signal)).rejects.toThrow(/not retried/);
77
104
  await expect(transport.request('type', {}, new AbortController().signal)).rejects.toThrow(/closed/);
78
105
  await transport.close();
79
106
  });
80
107
  it('cancels a real process and refuses already-cancelled requests', async () => {
81
- const transport = new HelperTransport(process.execPath, ['-e', 'process.stdin.resume()']);
108
+ const transport = helper('process.stdin.resume()');
82
109
  const abort = new AbortController();
83
110
  const promise = transport.request('observe', {}, abort.signal);
84
111
  abort.abort();
@@ -87,20 +114,50 @@ describe('native helper transport', () => {
87
114
  expect(transport.closed).toBe(true);
88
115
  });
89
116
  it('reports process death rather than leaving requests pending', async () => {
90
- const transport = new HelperTransport(process.execPath, ['-e', 'process.exit(3)']);
117
+ const transport = helper('process.exit(3)');
91
118
  await expect(transport.request('status', {}, new AbortController().signal)).rejects.toThrow(/exited/);
92
119
  await transport.close();
93
120
  });
94
121
  it('sends resume independently of a pending operation and permanently closes on stop', async () => {
95
- const transport = new HelperTransport(process.execPath, ['-e', `const lines=require('readline').createInterface({input:process.stdin});
122
+ const transport: HelperTransport = helper(`const lines=require('readline').createInterface({input:process.stdin});
96
123
  let request;
97
124
  lines.on('line',line=>{const r=JSON.parse(line);
98
125
  if(r.method){request=r;process.stdout.write(JSON.stringify({version:4,event:'control_state',id:r.id,state:'paused_by_user'})+'\\n');}
99
126
  else if(r.control==='resume'){process.stdout.write(JSON.stringify({version:4,id:request.id,ok:true,result:{resumed:true}})+'\\n');}
100
- });`], 1000, state => { if (state.state === 'paused_by_user') transport.control('resume'); });
127
+ });`, { timeoutMs: 1000, onEvent: (event) => { if (event.state === 'paused_by_user') transport.control('resume'); } });
101
128
  expect(await transport.request('click', {}, new AbortController().signal)).toEqual({resumed:true});
102
129
  transport.control('stop');
103
130
  await expect(transport.request('click', {}, new AbortController().signal)).rejects.toThrow(/closed/);
104
131
  await transport.close();
105
132
  });
106
133
  });
134
+
135
+ describe('helper events', () => {
136
+ it('delivers a registered uncorrelated event without disturbing the pending request', async () => {
137
+ const events: HelperEvent[] = [];
138
+ const transport = helper(`process.stdin.once('data', bytes => {
139
+ const r=JSON.parse(bytes.toString());
140
+ process.stdout.write(JSON.stringify({version:4,event:'cursor',x:10,y:20})+'\\n');
141
+ process.stdout.write(JSON.stringify({version:4,id:r.id,ok:true,result:{clicked:true}})+'\\n');
142
+ });`, { events: { cursor: cursorEvent }, onEvent: (event) => events.push(event) });
143
+ try {
144
+ expect(await transport.request('click', {}, new AbortController().signal)).toEqual({ clicked: true });
145
+ expect(events).toEqual([{ version: 4, event: 'cursor', x: 10, y: 20 }]);
146
+ } finally { await transport.close(); }
147
+ });
148
+ it('delivers events that arrive while no request is pending', async () => {
149
+ const received = new Promise<HelperEvent>((resolve) => {
150
+ const transport = helper(`process.stdout.write(JSON.stringify({version:4,event:'cursor',x:1,y:2})+'\\n'); process.stdin.resume();`,
151
+ { events: { cursor: cursorEvent }, onEvent: (event) => { resolve(event); void transport.close(); } });
152
+ });
153
+ expect(await received).toEqual({ version: 4, event: 'cursor', x: 1, y: 2 });
154
+ });
155
+ it('treats an unregistered or malformed event as a protocol failure', async () => {
156
+ for (const frame of ['{version:4,event:"teleport"}', '{version:4,event:"cursor",x:"left",y:2}']) {
157
+ const transport = helper(`process.stdin.once('data', () => process.stdout.write(JSON.stringify(${frame})+'\\n'));`,
158
+ { events: { cursor: cursorEvent } });
159
+ await expect(transport.request('click', {}, new AbortController().signal)).rejects.toThrow(/protocol/);
160
+ await transport.close();
161
+ }
162
+ });
163
+ });
@@ -1,7 +1,28 @@
1
1
  import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process';
2
2
  import { randomUUID } from 'node:crypto';
3
- import { JsonLineDecoder } from './protocol.js';
4
- import { MAX_FRAME_BYTES, PROTOCOL_VERSION, responseSchema, controlStateSchema, controlCommandSchema, type ControlState } from './contracts.js';
3
+ import { z } from 'zod';
4
+ import { JsonLineDecoder, MAX_FRAME_BYTES, TAKEOVER_PROTOCOL_VERSION, controlCommandSchema, controlStateSchemaFor, responseSchemaFor } from './protocol.js';
5
+
6
+ /** A validated frame the helper emitted on its own (state changes, cursor moves, preview frames). */
7
+ export interface HelperEvent { readonly event: string; readonly [field: string]: unknown }
8
+
9
+ export interface HelperTransportOptions {
10
+ readonly protocolVersion: number;
11
+ readonly timeoutMs?: number;
12
+ /** Schemas for uncorrelated events by name. `control_state` is built in; any other name fails closed. */
13
+ readonly events?: Readonly<Record<string, z.ZodType<HelperEvent>>>;
14
+ readonly onEvent?: (event: HelperEvent) => void;
15
+ }
16
+
17
+ const eventFrame = z.object({ event: z.string() }).passthrough();
18
+
19
+ /** A request the helper refused; `code` is the helper's machine-readable reason. */
20
+ export class HelperError extends Error {
21
+ constructor(readonly code: string, readonly detail: string) {
22
+ super(`${code}: ${detail}`);
23
+ this.name = 'HelperError';
24
+ }
25
+ }
5
26
 
6
27
  interface Pending {
7
28
  id: string;
@@ -22,9 +43,18 @@ export class HelperTransport {
22
43
  private stderrBytes = 0;
23
44
  get closed(): boolean { return this.stopped; }
24
45
  get stoppedByUser(): boolean { return this.userStopped; }
46
+ /** Settles once the helper process has exited, whatever the reason. */
47
+ get done(): Promise<void> { return this.exited; }
48
+
49
+ private readonly version: number;
50
+ private readonly timeoutMs: number;
25
51
 
26
- constructor(command: string, args: string[], private readonly timeoutMs = 15_000,
27
- private readonly onState: (state: ControlState) => void = () => undefined) {
52
+ constructor(command: string, args: string[], options: HelperTransportOptions) {
53
+ this.version = options.protocolVersion;
54
+ this.timeoutMs = options.timeoutMs ?? 15_000;
55
+ const onEvent = options.onEvent ?? (() => undefined);
56
+ const controlState = controlStateSchemaFor(this.version);
57
+ const response = responseSchemaFor(this.version);
28
58
  this.child = spawn(command, args, { shell: false, windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'] });
29
59
  this.exited = new Promise((resolve) => this.child.once('close', () => resolve()));
30
60
  const decoder = new JsonLineDecoder(MAX_FRAME_BYTES);
@@ -39,21 +69,28 @@ export class HelperTransport {
39
69
  this.child.stdout.on('data', (bytes: Buffer) => {
40
70
  try {
41
71
  for (const frame of decoder.push(bytes)) {
42
- const event = controlStateSchema.safeParse(frame);
43
- if (event.success) {
72
+ const tagged = eventFrame.safeParse(frame);
73
+ if (tagged.success && tagged.data.event === 'control_state') {
74
+ const state = controlState.parse(frame);
44
75
  const pending = this.pending;
45
- if (!pending || event.data.id !== pending.id) throw new Error('Unmatched control state');
46
- pending.waiting(event.data.state === 'waiting_for_focus' || event.data.state === 'paused_by_user');
47
- this.onState(event.data);
76
+ if (!pending || state.id !== pending.id) throw new Error('Unmatched control state');
77
+ pending.waiting(state.state === 'waiting_for_focus' || state.state === 'paused_by_user');
78
+ onEvent(state);
79
+ continue;
80
+ }
81
+ if (tagged.success) {
82
+ const schema = options.events?.[tagged.data.event];
83
+ if (!schema) throw new Error('Unregistered helper event');
84
+ onEvent(schema.parse(frame));
48
85
  continue;
49
86
  }
50
- const response = responseSchema.parse(frame);
87
+ const reply = response.parse(frame);
51
88
  const pending = this.pending;
52
- if (!pending || pending.id !== response.id) throw new Error('Unmatched response');
89
+ if (!pending || pending.id !== reply.id) throw new Error('Unmatched response');
53
90
  this.pending = undefined;
54
91
  pending.dispose();
55
- if (response.ok) pending.resolve(response.result);
56
- else pending.reject(new Error(`${response.error.code}: ${response.error.message}`));
92
+ if (reply.ok) pending.resolve(reply.result);
93
+ else pending.reject(new HelperError(reply.error.code, reply.error.message));
57
94
  }
58
95
  } catch {
59
96
  this.fail(new Error('Computer Use protocol mismatch or invalid response; update the extension. Action not retried.'));
@@ -76,7 +113,7 @@ export class HelperTransport {
76
113
  }
77
114
 
78
115
  /** Control uses the reader side of the helper, never its blocked request queue. */
79
- control(command: 'pause' | 'resume' | 'stop'): void {
116
+ control(command: 'pause' | 'resume' | 'stop' | 'takeover'): void {
80
117
  controlCommandSchema.parse(command);
81
118
  if (this.stopped) throw new Error('Computer Use connection closed');
82
119
  if (command === 'stop') {
@@ -84,14 +121,15 @@ export class HelperTransport {
84
121
  this.fail(new Error('Computer Use stopped by user; action not retried.'));
85
122
  return;
86
123
  }
87
- this.child.stdin.write(JSON.stringify({version: PROTOCOL_VERSION, control: command}) + '\n');
124
+ const sent = command === 'takeover' && this.version < TAKEOVER_PROTOCOL_VERSION ? 'pause' : command;
125
+ this.child.stdin.write(JSON.stringify({version: this.version, control: sent}) + '\n');
88
126
  }
89
127
 
90
128
  private send(method: string, params: unknown, signal: AbortSignal): Promise<unknown> {
91
129
  if (this.stopped) return Promise.reject(new Error('Computer Use connection closed; observe again with a new connection.'));
92
130
  if (signal.aborted) return Promise.reject(new Error('Computer Use cancelled'));
93
131
  const id = randomUUID();
94
- const frame = JSON.stringify({ version: PROTOCOL_VERSION, id, method, params }) + '\n';
132
+ const frame = JSON.stringify({ version: this.version, id, method, params }) + '\n';
95
133
  if (Buffer.byteLength(frame) > MAX_FRAME_BYTES) return Promise.reject(new Error('Computer Use request limit exceeded'));
96
134
  return new Promise((resolve, reject) => {
97
135
  const abort = () => this.fail(new Error('Computer Use cancelled; action not retried.'));