@moxxy/plugin-computer-control 0.41.2 → 0.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (269) hide show
  1. package/bin/win32-x64/moxxy-computer.exe +0 -0
  2. package/bin/win32-x64/moxxy-computer.exe.json +1 -1
  3. package/dist/backend/access.d.ts +129 -0
  4. package/dist/backend/access.d.ts.map +1 -0
  5. package/dist/backend/access.js +158 -0
  6. package/dist/backend/access.js.map +1 -0
  7. package/dist/backend/app-hints.d.ts +14 -0
  8. package/dist/backend/app-hints.d.ts.map +1 -0
  9. package/dist/backend/app-hints.js +30 -0
  10. package/dist/backend/app-hints.js.map +1 -0
  11. package/dist/backend/backend.d.ts +70 -0
  12. package/dist/backend/backend.d.ts.map +1 -0
  13. package/dist/backend/backend.js +466 -0
  14. package/dist/backend/backend.js.map +1 -0
  15. package/dist/backend/rpc.d.ts +1438 -0
  16. package/dist/backend/rpc.d.ts.map +1 -0
  17. package/dist/backend/rpc.js +75 -0
  18. package/dist/backend/rpc.js.map +1 -0
  19. package/dist/backend/turn-controls.d.ts +20 -0
  20. package/dist/backend/turn-controls.d.ts.map +1 -0
  21. package/dist/backend/turn-controls.js +115 -0
  22. package/dist/backend/turn-controls.js.map +1 -0
  23. package/dist/contract/guidance.d.ts +12 -0
  24. package/dist/contract/guidance.d.ts.map +1 -0
  25. package/dist/contract/guidance.js +42 -0
  26. package/dist/contract/guidance.js.map +1 -0
  27. package/dist/contract/image.d.ts +29 -0
  28. package/dist/contract/image.d.ts.map +1 -0
  29. package/dist/contract/image.js +46 -0
  30. package/dist/contract/image.js.map +1 -0
  31. package/dist/contract/keys.d.ts +15 -0
  32. package/dist/contract/keys.d.ts.map +1 -0
  33. package/dist/contract/keys.js +118 -0
  34. package/dist/contract/keys.js.map +1 -0
  35. package/dist/contract/outcome.d.ts +61 -0
  36. package/dist/contract/outcome.d.ts.map +1 -0
  37. package/dist/contract/outcome.js +63 -0
  38. package/dist/contract/outcome.js.map +1 -0
  39. package/dist/contract/progress.d.ts +27 -0
  40. package/dist/contract/progress.d.ts.map +1 -0
  41. package/dist/contract/progress.js +36 -0
  42. package/dist/contract/progress.js.map +1 -0
  43. package/dist/contract/tools.d.ts +338 -0
  44. package/dist/contract/tools.d.ts.map +1 -0
  45. package/dist/contract/tools.js +195 -0
  46. package/dist/contract/tools.js.map +1 -0
  47. package/dist/contract/untrusted.d.ts +3 -0
  48. package/dist/contract/untrusted.d.ts.map +1 -0
  49. package/dist/contract/untrusted.js +8 -0
  50. package/dist/contract/untrusted.js.map +1 -0
  51. package/dist/helper/artifact.d.ts +41 -0
  52. package/dist/helper/artifact.d.ts.map +1 -0
  53. package/dist/helper/artifact.js +189 -0
  54. package/dist/helper/artifact.js.map +1 -0
  55. package/dist/helper/protocol.d.ts +80 -0
  56. package/dist/helper/protocol.d.ts.map +1 -0
  57. package/dist/helper/protocol.js +49 -0
  58. package/dist/helper/protocol.js.map +1 -0
  59. package/dist/helper/transport.d.ts +43 -0
  60. package/dist/helper/transport.d.ts.map +1 -0
  61. package/dist/{windows → helper}/transport.js +45 -19
  62. package/dist/helper/transport.js.map +1 -0
  63. package/dist/index.d.ts +9 -22
  64. package/dist/index.d.ts.map +1 -1
  65. package/dist/index.js +35 -42
  66. package/dist/index.js.map +1 -1
  67. package/dist/jev/ladder.d.ts +32 -0
  68. package/dist/jev/ladder.d.ts.map +1 -0
  69. package/dist/jev/ladder.js +66 -0
  70. package/dist/jev/ladder.js.map +1 -0
  71. package/dist/jev/memory.d.ts +11 -0
  72. package/dist/jev/memory.d.ts.map +1 -0
  73. package/dist/jev/memory.js +16 -0
  74. package/dist/jev/memory.js.map +1 -0
  75. package/dist/jev/run.d.ts +93 -0
  76. package/dist/jev/run.d.ts.map +1 -0
  77. package/dist/jev/run.js +403 -0
  78. package/dist/jev/run.js.map +1 -0
  79. package/dist/jev/train.d.ts +97 -0
  80. package/dist/jev/train.d.ts.map +1 -0
  81. package/dist/jev/train.js +28 -0
  82. package/dist/jev/train.js.map +1 -0
  83. package/dist/linux/profile.d.ts +6 -0
  84. package/dist/linux/profile.d.ts.map +1 -0
  85. package/dist/linux/profile.js +21 -0
  86. package/dist/linux/profile.js.map +1 -0
  87. package/dist/macos/profile.d.ts +5 -0
  88. package/dist/macos/profile.d.ts.map +1 -0
  89. package/dist/macos/profile.js +17 -0
  90. package/dist/macos/profile.js.map +1 -0
  91. package/dist/preview/controller.d.ts +127 -0
  92. package/dist/preview/controller.d.ts.map +1 -0
  93. package/dist/preview/controller.js +203 -0
  94. package/dist/preview/controller.js.map +1 -0
  95. package/dist/preview/surface.d.ts +9 -0
  96. package/dist/preview/surface.d.ts.map +1 -0
  97. package/dist/preview/surface.js +50 -0
  98. package/dist/preview/surface.js.map +1 -0
  99. package/dist/windows/maintenance.d.ts +1 -1
  100. package/dist/windows/maintenance.d.ts.map +1 -1
  101. package/dist/windows/maintenance.js +6 -5
  102. package/dist/windows/maintenance.js.map +1 -1
  103. package/dist/windows/profile.d.ts +5 -0
  104. package/dist/windows/profile.d.ts.map +1 -0
  105. package/dist/windows/profile.js +17 -0
  106. package/dist/windows/profile.js.map +1 -0
  107. package/learned/README.md +26 -0
  108. package/learned/cases/com.apple.calculator.json +395 -0
  109. package/learned/cases/com.apple.finder.json +115 -0
  110. package/learned/cases/com.apple.safari.json +112 -0
  111. package/learned/cases/com.apple.systempreferences.json +255 -0
  112. package/learned/com.apple.calculator-eace95fc.json +334 -0
  113. package/learned/com.apple.finder-27cf6ce8.json +260 -0
  114. package/learned/com.apple.safari-7cd9df4f.json +143 -0
  115. package/learned/com.apple.systempreferences-02cf0b8b.json +535 -0
  116. package/package.json +13 -7
  117. package/scripts/promote-learned.mjs +8 -0
  118. package/scripts/summarize-trial.mjs +68 -0
  119. package/scripts/train-learned.mjs +64 -0
  120. package/skills/computer-apps/blender.md +29 -0
  121. package/skills/computer-apps/browsers.md +50 -0
  122. package/skills/computer-apps/design-tools.md +37 -0
  123. package/skills/computer-apps/finder.md +23 -0
  124. package/skills/computer-apps/office.md +46 -0
  125. package/skills/computer-apps/video-editors.md +44 -0
  126. package/skills/computer-control.md +154 -191
  127. package/src/backend/access.test.ts +158 -0
  128. package/src/backend/access.ts +168 -0
  129. package/src/backend/app-hints.test.ts +59 -0
  130. package/src/backend/app-hints.ts +39 -0
  131. package/src/backend/backend.test.ts +725 -0
  132. package/src/backend/backend.ts +482 -0
  133. package/src/backend/contract-helper.fixture.mjs +138 -0
  134. package/src/backend/helper.fixture.ts +38 -0
  135. package/src/backend/rpc.ts +90 -0
  136. package/src/backend/turn-controls.test.ts +178 -0
  137. package/src/backend/turn-controls.ts +120 -0
  138. package/src/contract/guidance.test.ts +75 -0
  139. package/src/contract/guidance.ts +48 -0
  140. package/src/contract/image.test.ts +74 -0
  141. package/src/contract/image.ts +54 -0
  142. package/src/contract/keys.test.ts +103 -0
  143. package/src/contract/keys.ts +117 -0
  144. package/src/contract/outcome.test.ts +71 -0
  145. package/src/contract/outcome.ts +72 -0
  146. package/src/contract/progress.test.ts +56 -0
  147. package/src/contract/progress.ts +43 -0
  148. package/src/contract/tools.test.ts +210 -0
  149. package/src/contract/tools.ts +212 -0
  150. package/src/contract/untrusted.test.ts +20 -0
  151. package/src/contract/untrusted.ts +8 -0
  152. package/src/helper/artifact.test.ts +131 -0
  153. package/src/helper/artifact.ts +182 -0
  154. package/src/helper/protocol.test.ts +29 -0
  155. package/src/helper/protocol.ts +50 -0
  156. package/src/{windows → helper}/transport.test.ts +76 -19
  157. package/src/{windows → helper}/transport.ts +54 -16
  158. package/src/index.test.ts +112 -0
  159. package/src/index.ts +36 -53
  160. package/src/jev/ladder.test.ts +113 -0
  161. package/src/jev/ladder.ts +88 -0
  162. package/src/jev/memory.ts +20 -0
  163. package/src/jev/run.test.ts +700 -0
  164. package/src/jev/run.ts +443 -0
  165. package/src/jev/train.test.ts +40 -0
  166. package/src/jev/train.ts +41 -0
  167. package/src/linux/helper.test.ts +481 -0
  168. package/src/linux/profile.ts +25 -0
  169. package/src/macos/helper.test.ts +988 -0
  170. package/src/macos/profile.ts +19 -0
  171. package/src/preview/controller.test.ts +308 -0
  172. package/src/preview/controller.ts +262 -0
  173. package/src/preview/surface.test.ts +68 -0
  174. package/src/preview/surface.ts +45 -0
  175. package/src/skill.test.ts +34 -0
  176. package/src/windows/maintenance.ts +8 -7
  177. package/src/windows/profile.ts +19 -0
  178. package/dist/shell.d.ts +0 -56
  179. package/dist/shell.d.ts.map +0 -1
  180. package/dist/shell.js +0 -189
  181. package/dist/shell.js.map +0 -1
  182. package/dist/temporary-files.d.ts +0 -2
  183. package/dist/temporary-files.d.ts.map +0 -1
  184. package/dist/temporary-files.js +0 -10
  185. package/dist/temporary-files.js.map +0 -1
  186. package/dist/tools/applescript.d.ts +0 -2
  187. package/dist/tools/applescript.d.ts.map +0 -1
  188. package/dist/tools/applescript.js +0 -50
  189. package/dist/tools/applescript.js.map +0 -1
  190. package/dist/tools/click.d.ts +0 -2
  191. package/dist/tools/click.d.ts.map +0 -1
  192. package/dist/tools/click.js +0 -58
  193. package/dist/tools/click.js.map +0 -1
  194. package/dist/tools/clipboard.d.ts +0 -2
  195. package/dist/tools/clipboard.d.ts.map +0 -1
  196. package/dist/tools/clipboard.js +0 -71
  197. package/dist/tools/clipboard.js.map +0 -1
  198. package/dist/tools/key.d.ts +0 -11
  199. package/dist/tools/key.d.ts.map +0 -1
  200. package/dist/tools/key.js +0 -143
  201. package/dist/tools/key.js.map +0 -1
  202. package/dist/tools/open.d.ts +0 -2
  203. package/dist/tools/open.d.ts.map +0 -1
  204. package/dist/tools/open.js +0 -91
  205. package/dist/tools/open.js.map +0 -1
  206. package/dist/tools/screenshot.d.ts +0 -2
  207. package/dist/tools/screenshot.d.ts.map +0 -1
  208. package/dist/tools/screenshot.js +0 -162
  209. package/dist/tools/screenshot.js.map +0 -1
  210. package/dist/tools/type.d.ts +0 -8
  211. package/dist/tools/type.d.ts.map +0 -1
  212. package/dist/tools/type.js +0 -65
  213. package/dist/tools/type.js.map +0 -1
  214. package/dist/windows/artifact.d.ts +0 -3
  215. package/dist/windows/artifact.d.ts.map +0 -1
  216. package/dist/windows/artifact.js +0 -57
  217. package/dist/windows/artifact.js.map +0 -1
  218. package/dist/windows/backend.d.ts +0 -11
  219. package/dist/windows/backend.d.ts.map +0 -1
  220. package/dist/windows/backend.js +0 -122
  221. package/dist/windows/backend.js.map +0 -1
  222. package/dist/windows/contracts.d.ts +0 -1627
  223. package/dist/windows/contracts.d.ts.map +0 -1
  224. package/dist/windows/contracts.js +0 -137
  225. package/dist/windows/contracts.js.map +0 -1
  226. package/dist/windows/control-service.d.ts +0 -12
  227. package/dist/windows/control-service.d.ts.map +0 -1
  228. package/dist/windows/control-service.js +0 -65
  229. package/dist/windows/control-service.js.map +0 -1
  230. package/dist/windows/guidance.d.ts +0 -3
  231. package/dist/windows/guidance.d.ts.map +0 -1
  232. package/dist/windows/guidance.js +0 -20
  233. package/dist/windows/guidance.js.map +0 -1
  234. package/dist/windows/protocol.d.ts +0 -9
  235. package/dist/windows/protocol.d.ts.map +0 -1
  236. package/dist/windows/protocol.js +0 -32
  237. package/dist/windows/protocol.js.map +0 -1
  238. package/dist/windows/transport.d.ts +0 -23
  239. package/dist/windows/transport.d.ts.map +0 -1
  240. package/dist/windows/transport.js.map +0 -1
  241. package/src/shell.test.ts +0 -186
  242. package/src/shell.ts +0 -213
  243. package/src/temporary-files.test.ts +0 -18
  244. package/src/temporary-files.ts +0 -9
  245. package/src/tools/applescript-serialize.test.ts +0 -74
  246. package/src/tools/applescript.ts +0 -53
  247. package/src/tools/click.ts +0 -60
  248. package/src/tools/clipboard.ts +0 -72
  249. package/src/tools/key.ts +0 -155
  250. package/src/tools/open.ts +0 -96
  251. package/src/tools/screenshot.test.ts +0 -137
  252. package/src/tools/screenshot.ts +0 -180
  253. package/src/tools/type.ts +0 -68
  254. package/src/tools.test.ts +0 -94
  255. package/src/windows/action-contracts.test.ts +0 -13
  256. package/src/windows/artifact.test.ts +0 -16
  257. package/src/windows/artifact.ts +0 -57
  258. package/src/windows/backend.test.ts +0 -41
  259. package/src/windows/backend.ts +0 -122
  260. package/src/windows/contracts.test.ts +0 -81
  261. package/src/windows/contracts.ts +0 -143
  262. package/src/windows/control-service.test.ts +0 -58
  263. package/src/windows/control-service.ts +0 -68
  264. package/src/windows/guidance.test.ts +0 -29
  265. package/src/windows/guidance.ts +0 -21
  266. package/src/windows/model-contract.test.ts +0 -37
  267. package/src/windows/protocol.ts +0 -27
  268. package/src/windows/text-contracts.test.ts +0 -14
  269. package/src/windows/window-typing.test.ts +0 -14
@@ -0,0 +1,535 @@
1
+ {
2
+ "app": "com.apple.systempreferences",
3
+ "targets": [
4
+ {
5
+ "do": "click",
6
+ "target": "drukarki i skanery left sidebar",
7
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#38/komórka:#1/tekst:com.apple.Print-Scan-Settings.extension",
8
+ "label": "tekst\u001fDrukarki i skanery",
9
+ "way": 0,
10
+ "uses": 2,
11
+ "at": 1790911224237,
12
+ "effect": [
13
+ "okno standardowe\u001fDrukarki i skanery"
14
+ ]
15
+ },
16
+ {
17
+ "do": "click",
18
+ "target": "kontrolery do gier left sidebar",
19
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#37/komórka:#1/tekst:com.apple.Game-Controller-Settings.extension",
20
+ "label": "tekst\u001fKontrolery do gier",
21
+ "way": 0,
22
+ "uses": 2,
23
+ "at": 1790911222681,
24
+ "effect": [
25
+ "okno standardowe\u001fKontrolery do gier"
26
+ ]
27
+ },
28
+ {
29
+ "do": "click",
30
+ "target": "gładzik left sidebar",
31
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#36/komórka:#1/tekst:com.apple.Trackpad-Settings.extension",
32
+ "label": "tekst\u001fGładzik",
33
+ "way": 0,
34
+ "uses": 2,
35
+ "at": 1790911221114,
36
+ "effect": [
37
+ "okno standardowe\u001fGładzik"
38
+ ]
39
+ },
40
+ {
41
+ "do": "click",
42
+ "target": "klawiatura left sidebar",
43
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#35/komórka:#1/tekst:com.apple.Keyboard-Settings.extension",
44
+ "label": "tekst\u001fKlawiatura",
45
+ "way": 0,
46
+ "uses": 2,
47
+ "at": 1790911219487,
48
+ "effect": [
49
+ "okno standardowe\u001fKlawiatura"
50
+ ]
51
+ },
52
+ {
53
+ "do": "click",
54
+ "target": "prywatność i ochrona left sidebar",
55
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#26/komórka:#1/tekst:com.apple.settings.PrivacySecurity.extension",
56
+ "label": "tekst\u001fPrywatność i ochrona",
57
+ "way": 0,
58
+ "uses": 2,
59
+ "at": 1790911217755,
60
+ "effect": [
61
+ "okno standardowe\u001fPrywatność i ochrona"
62
+ ]
63
+ },
64
+ {
65
+ "do": "click",
66
+ "target": "ekran blokady left sidebar",
67
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#25/komórka:#1/tekst:com.apple.Lock-Screen-Settings.extension",
68
+ "label": "tekst\u001fEkran blokady",
69
+ "way": 0,
70
+ "uses": 2,
71
+ "at": 1790911216003,
72
+ "effect": [
73
+ "okno standardowe\u001fEkran blokady"
74
+ ]
75
+ },
76
+ {
77
+ "do": "click",
78
+ "target": "czas przed ekranem left sidebar",
79
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#23/komórka:#1/tekst:com.apple.Screen-Time-Settings.extension",
80
+ "label": "tekst\u001fCzas przed ekranem",
81
+ "way": 0,
82
+ "uses": 2,
83
+ "at": 1790911214397,
84
+ "effect": [
85
+ "okno standardowe\u001fCzas przed ekranem"
86
+ ]
87
+ },
88
+ {
89
+ "do": "click",
90
+ "target": "tryb skupienia left sidebar",
91
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#22/komórka:#1/tekst:com.apple.Focus-Settings.extension",
92
+ "label": "tekst\u001fTryb skupienia",
93
+ "way": 0,
94
+ "uses": 2,
95
+ "at": 1790911212814,
96
+ "effect": [
97
+ "okno standardowe\u001fTryb skupienia"
98
+ ]
99
+ },
100
+ {
101
+ "do": "click",
102
+ "target": "dźwięk left sidebar",
103
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#21/komórka:#1/tekst:com.apple.Sound-Settings.extension",
104
+ "label": "tekst\u001fDźwięk",
105
+ "way": 0,
106
+ "uses": 2,
107
+ "at": 1790911211133,
108
+ "effect": [
109
+ "okno standardowe\u001fDźwięk"
110
+ ]
111
+ },
112
+ {
113
+ "do": "click",
114
+ "target": "powiadomienia left sidebar",
115
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#20/komórka:#1/tekst:com.apple.Notifications-Settings.extension",
116
+ "label": "tekst\u001fPowiadomienia",
117
+ "way": 0,
118
+ "uses": 2,
119
+ "at": 1790911209346,
120
+ "effect": [
121
+ "okno standardowe\u001fPowiadomienia"
122
+ ]
123
+ },
124
+ {
125
+ "do": "click",
126
+ "target": "wyświetlacze left sidebar",
127
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#18/komórka:#1/tekst:com.apple.Displays-Settings.extension",
128
+ "label": "tekst\u001fWyświetlacze",
129
+ "way": 0,
130
+ "uses": 2,
131
+ "at": 1790911207322,
132
+ "effect": [
133
+ "okno standardowe\u001fWyświetlacze"
134
+ ]
135
+ },
136
+ {
137
+ "do": "click",
138
+ "target": "wygląd left sidebar",
139
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#17/komórka:#1/tekst:com.apple.Appearance-Settings.extension",
140
+ "label": "tekst\u001fWygląd",
141
+ "way": 0,
142
+ "uses": 2,
143
+ "at": 1790911205685,
144
+ "effect": [
145
+ "okno standardowe\u001fWygląd"
146
+ ]
147
+ },
148
+ {
149
+ "do": "click",
150
+ "target": "tapeta left sidebar",
151
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#16/komórka:#1/tekst:com.apple.Wallpaper-Settings.extension",
152
+ "label": "tekst\u001fTapeta",
153
+ "way": 0,
154
+ "uses": 2,
155
+ "at": 1790911203993,
156
+ "effect": [
157
+ "okno standardowe\u001fTapeta"
158
+ ]
159
+ },
160
+ {
161
+ "do": "click",
162
+ "target": "spotlight left sidebar",
163
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#15/komórka:#1/tekst:com.apple.Spotlight-Settings.extension",
164
+ "label": "tekst\u001fSpotlight",
165
+ "way": 0,
166
+ "uses": 2,
167
+ "at": 1790911202001,
168
+ "effect": [
169
+ "okno standardowe\u001fSpotlight"
170
+ ]
171
+ },
172
+ {
173
+ "do": "click",
174
+ "target": "pasek menu left sidebar",
175
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#14/komórka:#1/tekst:com.apple.ControlCenter-Settings.extension",
176
+ "label": "tekst\u001fPasek menu",
177
+ "way": 0,
178
+ "uses": 2,
179
+ "at": 1790911200121
180
+ },
181
+ {
182
+ "do": "click",
183
+ "target": "dostępność left sidebar",
184
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#13/komórka:#1/tekst:com.apple.Accessibility-Settings.extension",
185
+ "label": "tekst\u001fDostępność",
186
+ "way": 0,
187
+ "uses": 2,
188
+ "at": 1790911197851,
189
+ "effect": [
190
+ "okno standardowe\u001fDostępność"
191
+ ]
192
+ },
193
+ {
194
+ "do": "click",
195
+ "target": "biurko i dock left sidebar",
196
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#12/komórka:#1/tekst:com.apple.Desktop-Settings.extension",
197
+ "label": "tekst\u001fBiurko i Dock",
198
+ "way": 0,
199
+ "uses": 2,
200
+ "at": 1790911196161,
201
+ "effect": [
202
+ "okno standardowe\u001fBiurko i Dock"
203
+ ]
204
+ },
205
+ {
206
+ "do": "click",
207
+ "target": "apple intelligence i siri left sidebar",
208
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#11/komórka:#1/tekst:com.apple.Siri-Settings.extension*siri-sae",
209
+ "label": "tekst\u001fApple Intelligence i Siri",
210
+ "way": 0,
211
+ "uses": 2,
212
+ "at": 1790911194315,
213
+ "effect": [
214
+ "okno standardowe\u001fApple Intelligence i Siri"
215
+ ]
216
+ },
217
+ {
218
+ "do": "click",
219
+ "target": "ogólne left sidebar",
220
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#10/komórka:#1/tekst:com.apple.systempreferences.GeneralSettings",
221
+ "label": "tekst\u001fOgólne",
222
+ "way": 0,
223
+ "uses": 2,
224
+ "at": 1790911192702
225
+ },
226
+ {
227
+ "do": "click",
228
+ "target": "bateria left sidebar",
229
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#8/komórka:#1/tekst:com.apple.Battery-Settings.extension*BatteryPreferences",
230
+ "label": "tekst\u001fBateria",
231
+ "way": 0,
232
+ "uses": 2,
233
+ "at": 1790911191166
234
+ },
235
+ {
236
+ "do": "click",
237
+ "target": "vpn left sidebar",
238
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#7/komórka:#1/tekst:com.apple.NetworkExtensionSettingsUI.NESettingsUIExtension*vpn",
239
+ "label": "tekst\u001fVPN",
240
+ "way": 0,
241
+ "uses": 2,
242
+ "at": 1790911189574,
243
+ "effect": [
244
+ "okno standardowe\u001fVPN"
245
+ ]
246
+ },
247
+ {
248
+ "do": "click",
249
+ "target": "sieć left sidebar",
250
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#6/komórka:#1/tekst:com.apple.Network-Settings.extension",
251
+ "label": "tekst\u001fSieć",
252
+ "way": 0,
253
+ "uses": 2,
254
+ "at": 1790911188092,
255
+ "effect": [
256
+ "okno standardowe\u001fSieć"
257
+ ]
258
+ },
259
+ {
260
+ "do": "click",
261
+ "target": "bluetooth left sidebar",
262
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#5/komórka:#1/tekst:com.apple.BluetoothSettings",
263
+ "label": "tekst\u001fBluetooth",
264
+ "way": 0,
265
+ "uses": 2,
266
+ "at": 1790911185549,
267
+ "effect": [
268
+ "okno standardowe\u001fBluetooth"
269
+ ]
270
+ },
271
+ {
272
+ "do": "click",
273
+ "target": "wi fi left sidebar",
274
+ "key": "okno standardowe:main/obszar przewijania:#1/lista:#1/wiersz zarysu:#4/komórka:#1/tekst:com.apple.wifi-settings-extension",
275
+ "label": "tekst\u001fWi‑Fi",
276
+ "way": 0,
277
+ "uses": 2,
278
+ "at": 1790911181887,
279
+ "effect": [
280
+ "okno standardowe\u001fWi‑Fi"
281
+ ]
282
+ },
283
+ {
284
+ "do": "click",
285
+ "target": "szukaj search field top sidebar",
286
+ "key": "okno standardowe:main/pole wyszukiwania tekstu:#1",
287
+ "label": "pole wyszukiwania tekstu\u001f",
288
+ "way": 0,
289
+ "uses": 1,
290
+ "at": 1790911179166
291
+ }
292
+ ],
293
+ "routes": [
294
+ {
295
+ "goal": "Open the Drukarki i skanery settings",
296
+ "steps": [
297
+ {
298
+ "do": "click",
299
+ "target": "Drukarki i skanery in the left sidebar",
300
+ "expect": "the Drukarki i skanery settings pane is shown"
301
+ }
302
+ ],
303
+ "uses": 1,
304
+ "at": 1790911224237
305
+ },
306
+ {
307
+ "goal": "Open the Kontrolery do gier settings",
308
+ "steps": [
309
+ {
310
+ "do": "click",
311
+ "target": "Kontrolery do gier in the left sidebar",
312
+ "expect": "the Kontrolery do gier settings pane is shown"
313
+ }
314
+ ],
315
+ "uses": 1,
316
+ "at": 1790911222681
317
+ },
318
+ {
319
+ "goal": "Open the Gładzik settings",
320
+ "steps": [
321
+ {
322
+ "do": "click",
323
+ "target": "Gładzik in the left sidebar",
324
+ "expect": "the Gładzik settings pane is shown"
325
+ }
326
+ ],
327
+ "uses": 1,
328
+ "at": 1790911221114
329
+ },
330
+ {
331
+ "goal": "Open the Klawiatura settings",
332
+ "steps": [
333
+ {
334
+ "do": "click",
335
+ "target": "Klawiatura in the left sidebar",
336
+ "expect": "the Klawiatura settings pane is shown"
337
+ }
338
+ ],
339
+ "uses": 1,
340
+ "at": 1790911219487
341
+ },
342
+ {
343
+ "goal": "Open the Prywatność i ochrona settings",
344
+ "steps": [
345
+ {
346
+ "do": "click",
347
+ "target": "Prywatność i ochrona in the left sidebar",
348
+ "expect": "the Prywatność i ochrona settings pane is shown"
349
+ }
350
+ ],
351
+ "uses": 1,
352
+ "at": 1790911217755
353
+ },
354
+ {
355
+ "goal": "Open the Ekran blokady settings",
356
+ "steps": [
357
+ {
358
+ "do": "click",
359
+ "target": "Ekran blokady in the left sidebar",
360
+ "expect": "the Ekran blokady settings pane is shown"
361
+ }
362
+ ],
363
+ "uses": 1,
364
+ "at": 1790911216003
365
+ },
366
+ {
367
+ "goal": "Open the Czas przed ekranem settings",
368
+ "steps": [
369
+ {
370
+ "do": "click",
371
+ "target": "Czas przed ekranem in the left sidebar",
372
+ "expect": "the Czas przed ekranem settings pane is shown"
373
+ }
374
+ ],
375
+ "uses": 1,
376
+ "at": 1790911214397
377
+ },
378
+ {
379
+ "goal": "Open the Tryb skupienia settings",
380
+ "steps": [
381
+ {
382
+ "do": "click",
383
+ "target": "Tryb skupienia in the left sidebar",
384
+ "expect": "the Tryb skupienia settings pane is shown"
385
+ }
386
+ ],
387
+ "uses": 1,
388
+ "at": 1790911212814
389
+ },
390
+ {
391
+ "goal": "Open the Dźwięk settings",
392
+ "steps": [
393
+ {
394
+ "do": "click",
395
+ "target": "Dźwięk in the left sidebar",
396
+ "expect": "the Dźwięk settings pane is shown"
397
+ }
398
+ ],
399
+ "uses": 1,
400
+ "at": 1790911211133
401
+ },
402
+ {
403
+ "goal": "Open the Powiadomienia settings",
404
+ "steps": [
405
+ {
406
+ "do": "click",
407
+ "target": "Powiadomienia in the left sidebar",
408
+ "expect": "the Powiadomienia settings pane is shown"
409
+ }
410
+ ],
411
+ "uses": 1,
412
+ "at": 1790911209346
413
+ },
414
+ {
415
+ "goal": "Open the Wyświetlacze settings",
416
+ "steps": [
417
+ {
418
+ "do": "click",
419
+ "target": "Wyświetlacze in the left sidebar",
420
+ "expect": "the Wyświetlacze settings pane is shown"
421
+ }
422
+ ],
423
+ "uses": 1,
424
+ "at": 1790911207322
425
+ },
426
+ {
427
+ "goal": "Open the Wygląd settings",
428
+ "steps": [
429
+ {
430
+ "do": "click",
431
+ "target": "Wygląd in the left sidebar",
432
+ "expect": "the Wygląd settings pane is shown"
433
+ }
434
+ ],
435
+ "uses": 1,
436
+ "at": 1790911205685
437
+ },
438
+ {
439
+ "goal": "Open the Tapeta settings",
440
+ "steps": [
441
+ {
442
+ "do": "click",
443
+ "target": "Tapeta in the left sidebar",
444
+ "expect": "the Tapeta settings pane is shown"
445
+ }
446
+ ],
447
+ "uses": 1,
448
+ "at": 1790911203993
449
+ },
450
+ {
451
+ "goal": "Open the Spotlight settings",
452
+ "steps": [
453
+ {
454
+ "do": "click",
455
+ "target": "Spotlight in the left sidebar",
456
+ "expect": "the Spotlight settings pane is shown"
457
+ }
458
+ ],
459
+ "uses": 1,
460
+ "at": 1790911202001
461
+ },
462
+ {
463
+ "goal": "Open the Pasek menu settings",
464
+ "steps": [
465
+ {
466
+ "do": "click",
467
+ "target": "Pasek menu in the left sidebar",
468
+ "expect": "the Pasek menu settings pane is shown"
469
+ }
470
+ ],
471
+ "uses": 1,
472
+ "at": 1790911200121
473
+ },
474
+ {
475
+ "goal": "Open the Dostępność settings",
476
+ "steps": [
477
+ {
478
+ "do": "click",
479
+ "target": "Dostępność in the left sidebar",
480
+ "expect": "the Dostępność settings pane is shown"
481
+ }
482
+ ],
483
+ "uses": 1,
484
+ "at": 1790911197851
485
+ },
486
+ {
487
+ "goal": "Open the Biurko i Dock settings",
488
+ "steps": [
489
+ {
490
+ "do": "click",
491
+ "target": "Biurko i Dock in the left sidebar",
492
+ "expect": "the Biurko i Dock settings pane is shown"
493
+ }
494
+ ],
495
+ "uses": 1,
496
+ "at": 1790911196161
497
+ },
498
+ {
499
+ "goal": "Open the Apple Intelligence i Siri settings",
500
+ "steps": [
501
+ {
502
+ "do": "click",
503
+ "target": "Apple Intelligence i Siri in the left sidebar",
504
+ "expect": "the Apple Intelligence i Siri settings pane is shown"
505
+ }
506
+ ],
507
+ "uses": 1,
508
+ "at": 1790911194315
509
+ },
510
+ {
511
+ "goal": "Open the Ogólne settings",
512
+ "steps": [
513
+ {
514
+ "do": "click",
515
+ "target": "Ogólne in the left sidebar",
516
+ "expect": "the Ogólne settings pane is shown"
517
+ }
518
+ ],
519
+ "uses": 1,
520
+ "at": 1790911192702
521
+ },
522
+ {
523
+ "goal": "Open the Bateria settings",
524
+ "steps": [
525
+ {
526
+ "do": "click",
527
+ "target": "Bateria in the left sidebar",
528
+ "expect": "the Bateria settings pane is shown"
529
+ }
530
+ ],
531
+ "uses": 1,
532
+ "at": 1790911191166
533
+ }
534
+ ]
535
+ }
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@moxxy/plugin-computer-control",
3
- "version": "0.41.2",
4
- "description": "Permission-gated computer control with macOS tools and an independent Windows x64 native backend.",
3
+ "version": "0.42.0",
4
+ "description": "Permission-gated control of desktop applications through bundled native helpers for macOS, Windows x64 and Linux (X11).",
5
5
  "keywords": [
6
6
  "moxxy",
7
7
  "agent",
@@ -41,6 +41,8 @@
41
41
  "dist",
42
42
  "src",
43
43
  "skills",
44
+ "learned",
45
+ "scripts",
44
46
  "bin"
45
47
  ],
46
48
  "moxxy": {
@@ -52,17 +54,21 @@
52
54
  },
53
55
  "dependencies": {
54
56
  "zod": "^3.25.76",
55
- "@moxxy/sdk": "0.41.2"
57
+ "@moxxy/sdk": "0.42.0",
58
+ "@moxxy/jev": "0.2.0"
56
59
  },
57
60
  "devDependencies": {
58
- "@types/node": "^22.10.0",
61
+ "@types/node": "^22.20.5",
59
62
  "typescript": "^6.0.3",
60
- "vitest": "^4.1.11",
61
- "@moxxy/tsconfig": "0.0.0",
62
- "@moxxy/vitest-preset": "0.0.0"
63
+ "vitest": "^5.0.3",
64
+ "@moxxy/vitest-preset": "0.0.0",
65
+ "@moxxy/tsconfig": "0.0.0"
63
66
  },
64
67
  "scripts": {
65
68
  "build": "tsc -p tsconfig.json",
69
+ "learned:promote": "node scripts/promote-learned.mjs",
70
+ "learned:train": "node scripts/train-learned.mjs",
71
+ "trial:summarize": "node scripts/summarize-trial.mjs",
66
72
  "typecheck": "tsc -p tsconfig.json --noEmit",
67
73
  "test": "vitest run",
68
74
  "clean": "rm -rf dist .turbo"
@@ -0,0 +1,8 @@
1
+ // Adds what this computer learned about apps to the lessons that ship with the plugin (`learned/`).
2
+ // usage: pnpm --filter @moxxy/plugin-computer-control learned:promote [directory]
3
+ import { moxxyPath } from '@moxxy/sdk/server';
4
+ import { promote, shippedLearned } from '../dist/jev/memory.js';
5
+
6
+ const from = process.argv[2] ?? moxxyPath('computer-use', 'learned');
7
+ const apps = await promote(from, shippedLearned);
8
+ console.log(apps.length === 0 ? `Nothing learned in ${from}.` : `Shipped lessons updated for: ${apps.join(', ')}\nRead the diff of ${shippedLearned} before committing: targets and routes are stored as written.`);
@@ -0,0 +1,68 @@
1
+ // What one live trial of Computer Use cost and where its time went.
2
+ // A trial is `moxxy -p "<task>" --allow-all --output-format json > <dir>/events.json`, run with
3
+ // MOXXY_JEV_TRACE=<dir>/jev.ndjson and MOXXY_COMPUTER_TIMING=<dir>/helper-timing.ndjson.
4
+ // usage: pnpm --filter @moxxy/plugin-computer-control trial:summarize <dir>
5
+ import { existsSync, readFileSync } from 'node:fs';
6
+ import { join } from 'node:path';
7
+
8
+ const dir = process.argv[2];
9
+ if (!dir) {
10
+ console.error('usage: summarize-trial.mjs <trial directory>');
11
+ process.exit(64);
12
+ }
13
+ const read = (name) => (existsSync(join(dir, name)) ? readFileSync(join(dir, name), 'utf8') : '');
14
+ const lines = (name) => read(name).trim().split('\n').filter(Boolean).map((line) => JSON.parse(line));
15
+ const s = (ms) => `${(ms / 1000).toFixed(1)}s`;
16
+ const sum = (values) => values.reduce((total, value) => total + value, 0);
17
+ const avg = (values) => (values.length ? Math.round(sum(values) / values.length) : 0);
18
+
19
+ const events = JSON.parse(read('events.json') || '[]');
20
+ const responses = events.filter((event) => event.type === 'provider_response');
21
+ const calls = events.filter((event) => event.type === 'tool_call_requested');
22
+ const results = new Map(events.filter((event) => event.type === 'tool_result').map((event) => [event.callId, event]));
23
+ const turn = (events.at(-1)?.ts ?? 0) - (events[0]?.ts ?? 0);
24
+ const modelMs = sum(responses.map((response) => response.timing?.providerWaitMs ?? 0));
25
+ const toolMs = sum(calls.map((call) => (results.get(call.callId)?.ts ?? call.ts) - call.ts));
26
+ console.log(`turn ${s(turn)} | model: ${responses.length} calls, ${s(modelMs)} | tools: ${calls.length} calls, ${s(toolMs)}`);
27
+ const counted = {};
28
+ for (const call of calls) counted[call.name] = (counted[call.name] ?? 0) + 1;
29
+ console.log(`tools: ${Object.entries(counted).sort((a, b) => b[1] - a[1]).map(([name, count]) => `${name}×${count}`).join(' ')}`);
30
+
31
+ const text = (output) => {
32
+ if (typeof output === 'string') return output;
33
+ const parts = Array.isArray(output) ? output : Array.isArray(output?.content) ? output.content : [output];
34
+ return parts.map((part) => (typeof part === 'string' ? part : part?.text ?? part?.forModel ?? '')).join('\n');
35
+ };
36
+ const traced = lines('jev.ndjson');
37
+ const runs = traced.filter((line) => line.run).map((line) => line.run);
38
+ for (const [index, call] of calls.filter((candidate) => candidate.name === 'computer_run').entries()) {
39
+ const result = results.get(call.callId);
40
+ const report = result ? text(result.output).split('\n').filter((line) => /^computer_run:|^\d+\. |^Not run/.test(line)).join('\n ') : '(no result)';
41
+ const time = runs[index]?.time;
42
+ const split = time ? ` [jev ${s(time.jev)}, actions ${s(time.act)}, looks ${s(time.look)}]` : '';
43
+ console.log(`run ${index + 1} (${call.input.app}, ${call.input.steps.length} steps)${split}: ${call.input.goal}\n ${report}`);
44
+ }
45
+
46
+ const asks = traced.filter((line) => !line.run);
47
+ if (asks.length > 0) {
48
+ console.log(`jev: ${asks.length} requests, avg ${avg(asks.map((ask) => ask.ms))} ms, avg ${avg(asks.map((ask) => ask.elements))} elements, errors ${asks.filter((ask) => ask.error).length}`);
49
+ }
50
+ if (runs.length > 0) {
51
+ const part = (name) => s(sum(runs.map((run) => run.time[name])));
52
+ console.log(`computer_run total: ${s(sum(runs.map((run) => run.ms)))} = jev ${part('jev')} + actions ${part('act')} + looks ${part('look')} + code`);
53
+ }
54
+
55
+ // Helper marks: the time since the mark before, by what the helper was doing. `begin` marks are the gaps between requests.
56
+ const phases = new Map();
57
+ for (const line of read('helper-timing.ndjson').split('\n')) {
58
+ const mark = /^\s*(-?\d+) ms\s+(.+)$/.exec(line);
59
+ if (!mark || /: begin$/.test(mark[2])) continue;
60
+ const phase = phases.get(mark[2]) ?? { ms: 0, count: 0 };
61
+ phases.set(mark[2], { ms: phase.ms + Number(mark[1]), count: phase.count + 1 });
62
+ }
63
+ if (phases.size > 0) {
64
+ const sorted = [...phases].sort((a, b) => b[1].ms - a[1].ms);
65
+ console.log(`helper ${s(sum(sorted.map(([, phase]) => phase.ms)))}: ${sorted.slice(0, 6).map(([name, phase]) => `${name} ${s(phase.ms)}×${phase.count}`).join(', ')}`);
66
+ }
67
+ const answer = events.filter((event) => event.type === 'assistant_message').at(-1);
68
+ console.log(`answer: ${(answer?.content ?? '(none)').replace(/\s+/g, ' ').slice(0, 900)}`);