@xagent-ai/cli 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (537) hide show
  1. package/.eslintrc.js +25 -0
  2. package/.gitmodules +3 -0
  3. package/.prettierrc.json +8 -0
  4. package/CONTRIBUTING.md +167 -0
  5. package/LICENSE +21 -0
  6. package/README.md +280 -0
  7. package/README_CN.md +280 -0
  8. package/dist/agents.d.ts +21 -0
  9. package/dist/agents.d.ts.map +1 -0
  10. package/dist/agents.js +463 -0
  11. package/dist/agents.js.map +1 -0
  12. package/dist/ai-client.d.ts +83 -0
  13. package/dist/ai-client.d.ts.map +1 -0
  14. package/dist/ai-client.js +1280 -0
  15. package/dist/ai-client.js.map +1 -0
  16. package/dist/auth.d.ts +25 -0
  17. package/dist/auth.d.ts.map +1 -0
  18. package/dist/auth.js +573 -0
  19. package/dist/auth.js.map +1 -0
  20. package/dist/cancellation.d.ts +46 -0
  21. package/dist/cancellation.d.ts.map +1 -0
  22. package/dist/cancellation.js +154 -0
  23. package/dist/cancellation.js.map +1 -0
  24. package/dist/checkpoint.d.ts +28 -0
  25. package/dist/checkpoint.d.ts.map +1 -0
  26. package/dist/checkpoint.js +186 -0
  27. package/dist/checkpoint.js.map +1 -0
  28. package/dist/cli.d.ts +3 -0
  29. package/dist/cli.d.ts.map +1 -0
  30. package/dist/cli.js +364 -0
  31. package/dist/cli.js.map +1 -0
  32. package/dist/config.d.ts +49 -0
  33. package/dist/config.d.ts.map +1 -0
  34. package/dist/config.js +205 -0
  35. package/dist/config.js.map +1 -0
  36. package/dist/context-compressor.d.ts +51 -0
  37. package/dist/context-compressor.d.ts.map +1 -0
  38. package/dist/context-compressor.js +231 -0
  39. package/dist/context-compressor.js.map +1 -0
  40. package/dist/conversation.d.ts +34 -0
  41. package/dist/conversation.d.ts.map +1 -0
  42. package/dist/conversation.js +221 -0
  43. package/dist/conversation.js.map +1 -0
  44. package/dist/gui-subagent/action-parser/actionParser.d.ts +19 -0
  45. package/dist/gui-subagent/action-parser/actionParser.d.ts.map +1 -0
  46. package/dist/gui-subagent/action-parser/actionParser.js +203 -0
  47. package/dist/gui-subagent/action-parser/actionParser.js.map +1 -0
  48. package/dist/gui-subagent/action-parser/constants.d.ts +8 -0
  49. package/dist/gui-subagent/action-parser/constants.d.ts.map +1 -0
  50. package/dist/gui-subagent/action-parser/constants.js +12 -0
  51. package/dist/gui-subagent/action-parser/constants.js.map +1 -0
  52. package/dist/gui-subagent/action-parser/index.d.ts +3 -0
  53. package/dist/gui-subagent/action-parser/index.d.ts.map +1 -0
  54. package/dist/gui-subagent/action-parser/index.js +6 -0
  55. package/dist/gui-subagent/action-parser/index.js.map +1 -0
  56. package/dist/gui-subagent/action-parser/types.d.ts +24 -0
  57. package/dist/gui-subagent/action-parser/types.d.ts.map +1 -0
  58. package/dist/gui-subagent/action-parser/types.js +12 -0
  59. package/dist/gui-subagent/action-parser/types.js.map +1 -0
  60. package/dist/gui-subagent/agent/gui-agent.d.ts +126 -0
  61. package/dist/gui-subagent/agent/gui-agent.d.ts.map +1 -0
  62. package/dist/gui-subagent/agent/gui-agent.js +820 -0
  63. package/dist/gui-subagent/agent/gui-agent.js.map +1 -0
  64. package/dist/gui-subagent/agent/index.d.ts +5 -0
  65. package/dist/gui-subagent/agent/index.d.ts.map +1 -0
  66. package/dist/gui-subagent/agent/index.js +5 -0
  67. package/dist/gui-subagent/agent/index.js.map +1 -0
  68. package/dist/gui-subagent/index.d.ts +43 -0
  69. package/dist/gui-subagent/index.d.ts.map +1 -0
  70. package/dist/gui-subagent/index.js +96 -0
  71. package/dist/gui-subagent/index.js.map +1 -0
  72. package/dist/gui-subagent/operator/base-operator.d.ts +108 -0
  73. package/dist/gui-subagent/operator/base-operator.d.ts.map +1 -0
  74. package/dist/gui-subagent/operator/base-operator.js +172 -0
  75. package/dist/gui-subagent/operator/base-operator.js.map +1 -0
  76. package/dist/gui-subagent/operator/browser-operator.d.ts +36 -0
  77. package/dist/gui-subagent/operator/browser-operator.d.ts.map +1 -0
  78. package/dist/gui-subagent/operator/browser-operator.js +306 -0
  79. package/dist/gui-subagent/operator/browser-operator.js.map +1 -0
  80. package/dist/gui-subagent/operator/computer-operator.d.ts +31 -0
  81. package/dist/gui-subagent/operator/computer-operator.d.ts.map +1 -0
  82. package/dist/gui-subagent/operator/computer-operator.js +441 -0
  83. package/dist/gui-subagent/operator/computer-operator.js.map +1 -0
  84. package/dist/gui-subagent/operator/desktop-operator.d.ts +55 -0
  85. package/dist/gui-subagent/operator/desktop-operator.d.ts.map +1 -0
  86. package/dist/gui-subagent/operator/desktop-operator.js +527 -0
  87. package/dist/gui-subagent/operator/desktop-operator.js.map +1 -0
  88. package/dist/gui-subagent/operator/index.d.ts +7 -0
  89. package/dist/gui-subagent/operator/index.d.ts.map +1 -0
  90. package/dist/gui-subagent/operator/index.js +6 -0
  91. package/dist/gui-subagent/operator/index.js.map +1 -0
  92. package/dist/gui-subagent/types/actions.d.ts +108 -0
  93. package/dist/gui-subagent/types/actions.d.ts.map +1 -0
  94. package/dist/gui-subagent/types/actions.js +39 -0
  95. package/dist/gui-subagent/types/actions.js.map +1 -0
  96. package/dist/gui-subagent/types/index.d.ts +6 -0
  97. package/dist/gui-subagent/types/index.d.ts.map +1 -0
  98. package/dist/gui-subagent/types/index.js +6 -0
  99. package/dist/gui-subagent/types/index.js.map +1 -0
  100. package/dist/gui-subagent/types/operator.d.ts +95 -0
  101. package/dist/gui-subagent/types/operator.d.ts.map +1 -0
  102. package/dist/gui-subagent/types/operator.js +16 -0
  103. package/dist/gui-subagent/types/operator.js.map +1 -0
  104. package/dist/gui-subagent/utils.d.ts +19 -0
  105. package/dist/gui-subagent/utils.d.ts.map +1 -0
  106. package/dist/gui-subagent/utils.js +42 -0
  107. package/dist/gui-subagent/utils.js.map +1 -0
  108. package/dist/hook.d.ts +73 -0
  109. package/dist/hook.d.ts.map +1 -0
  110. package/dist/hook.js +156 -0
  111. package/dist/hook.js.map +1 -0
  112. package/dist/index.d.ts +19 -0
  113. package/dist/index.d.ts.map +1 -0
  114. package/dist/index.js +19 -0
  115. package/dist/index.js.map +1 -0
  116. package/dist/input-history.d.ts +24 -0
  117. package/dist/input-history.d.ts.map +1 -0
  118. package/dist/input-history.js +94 -0
  119. package/dist/input-history.js.map +1 -0
  120. package/dist/input-processor.d.ts +31 -0
  121. package/dist/input-processor.d.ts.map +1 -0
  122. package/dist/input-processor.js +233 -0
  123. package/dist/input-processor.js.map +1 -0
  124. package/dist/keyboard-manager.d.ts +151 -0
  125. package/dist/keyboard-manager.d.ts.map +1 -0
  126. package/dist/keyboard-manager.js +396 -0
  127. package/dist/keyboard-manager.js.map +1 -0
  128. package/dist/logger.d.ts +75 -0
  129. package/dist/logger.d.ts.map +1 -0
  130. package/dist/logger.js +339 -0
  131. package/dist/logger.js.map +1 -0
  132. package/dist/mcp.d.ts +57 -0
  133. package/dist/mcp.d.ts.map +1 -0
  134. package/dist/mcp.js +483 -0
  135. package/dist/mcp.js.map +1 -0
  136. package/dist/memory.d.ts +25 -0
  137. package/dist/memory.d.ts.map +1 -0
  138. package/dist/memory.js +250 -0
  139. package/dist/memory.js.map +1 -0
  140. package/dist/print-system-prompt.d.ts +2 -0
  141. package/dist/print-system-prompt.d.ts.map +1 -0
  142. package/dist/print-system-prompt.js +40 -0
  143. package/dist/print-system-prompt.js.map +1 -0
  144. package/dist/session-manager.d.ts +41 -0
  145. package/dist/session-manager.d.ts.map +1 -0
  146. package/dist/session-manager.js +234 -0
  147. package/dist/session-manager.js.map +1 -0
  148. package/dist/session.d.ts +77 -0
  149. package/dist/session.d.ts.map +1 -0
  150. package/dist/session.js +1081 -0
  151. package/dist/session.js.map +1 -0
  152. package/dist/skill-invoker.d.ts +177 -0
  153. package/dist/skill-invoker.d.ts.map +1 -0
  154. package/dist/skill-invoker.js +1643 -0
  155. package/dist/skill-invoker.js.map +1 -0
  156. package/dist/skill-loader.d.ts +76 -0
  157. package/dist/skill-loader.d.ts.map +1 -0
  158. package/dist/skill-loader.js +407 -0
  159. package/dist/skill-loader.js.map +1 -0
  160. package/dist/slash-commands.d.ts +60 -0
  161. package/dist/slash-commands.d.ts.map +1 -0
  162. package/dist/slash-commands.js +1021 -0
  163. package/dist/slash-commands.js.map +1 -0
  164. package/dist/smart-approval.d.ts +137 -0
  165. package/dist/smart-approval.d.ts.map +1 -0
  166. package/dist/smart-approval.js +512 -0
  167. package/dist/smart-approval.js.map +1 -0
  168. package/dist/system-prompt-generator.d.ts +35 -0
  169. package/dist/system-prompt-generator.d.ts.map +1 -0
  170. package/dist/system-prompt-generator.js +729 -0
  171. package/dist/system-prompt-generator.js.map +1 -0
  172. package/dist/test-boundary-conditions.d.ts.map +1 -0
  173. package/dist/test-boundary-conditions.js.map +1 -0
  174. package/dist/test-cancellation-fix.d.ts.map +1 -0
  175. package/dist/test-cancellation-fix.js.map +1 -0
  176. package/dist/test-input-history.d.ts.map +1 -0
  177. package/dist/test-input-history.js.map +1 -0
  178. package/dist/test-interaction-flow.d.ts.map +1 -0
  179. package/dist/test-interaction-flow.js.map +1 -0
  180. package/dist/test-quick.d.ts.map +1 -0
  181. package/dist/test-quick.js.map +1 -0
  182. package/dist/test-user-interaction.d.ts.map +1 -0
  183. package/dist/test-user-interaction.js.map +1 -0
  184. package/dist/theme.d.ts +353 -0
  185. package/dist/theme.d.ts.map +1 -0
  186. package/dist/theme.js +383 -0
  187. package/dist/theme.js.map +1 -0
  188. package/dist/tools.d.ts +373 -0
  189. package/dist/tools.d.ts.map +1 -0
  190. package/dist/tools.js +2906 -0
  191. package/dist/tools.js.map +1 -0
  192. package/dist/types.d.ts +180 -0
  193. package/dist/types.d.ts.map +1 -0
  194. package/dist/types.js +23 -0
  195. package/dist/types.js.map +1 -0
  196. package/dist/unified-session.d.ts +42 -0
  197. package/dist/unified-session.d.ts.map +1 -0
  198. package/dist/unified-session.js +271 -0
  199. package/dist/unified-session.js.map +1 -0
  200. package/dist/update.d.ts +30 -0
  201. package/dist/update.d.ts.map +1 -0
  202. package/dist/update.js +211 -0
  203. package/dist/update.js.map +1 -0
  204. package/dist/workflow.d.ts +53 -0
  205. package/dist/workflow.d.ts.map +1 -0
  206. package/dist/workflow.js +405 -0
  207. package/dist/workflow.js.map +1 -0
  208. package/docs/architecture/mcp-integration-guide.md +131 -0
  209. package/docs/architecture/overview.md +93 -0
  210. package/docs/architecture/tool-system-design.md +89 -0
  211. package/docs/cli/commands.md +189 -0
  212. package/docs/smart-mode.md +257 -0
  213. package/docs/third-party-models.md +449 -0
  214. package/package.json +85 -0
  215. package/scripts/init-skills-path.js +58 -0
  216. package/skills/.claude-plugin/marketplace.json +45 -0
  217. package/skills/README.md +94 -0
  218. package/skills/THIRD_PARTY_NOTICES.md +405 -0
  219. package/skills/skills/algorithmic-art/LICENSE.txt +202 -0
  220. package/skills/skills/algorithmic-art/SKILL.md +405 -0
  221. package/skills/skills/algorithmic-art/templates/generator_template.js +223 -0
  222. package/skills/skills/algorithmic-art/templates/viewer.html +599 -0
  223. package/skills/skills/brand-guidelines/LICENSE.txt +202 -0
  224. package/skills/skills/brand-guidelines/SKILL.md +73 -0
  225. package/skills/skills/canvas-design/LICENSE.txt +202 -0
  226. package/skills/skills/canvas-design/SKILL.md +130 -0
  227. package/skills/skills/canvas-design/canvas-fonts/ArsenalSC-OFL.txt +93 -0
  228. package/skills/skills/canvas-design/canvas-fonts/ArsenalSC-Regular.ttf +0 -0
  229. package/skills/skills/canvas-design/canvas-fonts/BigShoulders-Bold.ttf +0 -0
  230. package/skills/skills/canvas-design/canvas-fonts/BigShoulders-OFL.txt +93 -0
  231. package/skills/skills/canvas-design/canvas-fonts/BigShoulders-Regular.ttf +0 -0
  232. package/skills/skills/canvas-design/canvas-fonts/Boldonse-OFL.txt +93 -0
  233. package/skills/skills/canvas-design/canvas-fonts/Boldonse-Regular.ttf +0 -0
  234. package/skills/skills/canvas-design/canvas-fonts/BricolageGrotesque-Bold.ttf +0 -0
  235. package/skills/skills/canvas-design/canvas-fonts/BricolageGrotesque-OFL.txt +93 -0
  236. package/skills/skills/canvas-design/canvas-fonts/BricolageGrotesque-Regular.ttf +0 -0
  237. package/skills/skills/canvas-design/canvas-fonts/CrimsonPro-Bold.ttf +0 -0
  238. package/skills/skills/canvas-design/canvas-fonts/CrimsonPro-Italic.ttf +0 -0
  239. package/skills/skills/canvas-design/canvas-fonts/CrimsonPro-OFL.txt +93 -0
  240. package/skills/skills/canvas-design/canvas-fonts/CrimsonPro-Regular.ttf +0 -0
  241. package/skills/skills/canvas-design/canvas-fonts/DMMono-OFL.txt +93 -0
  242. package/skills/skills/canvas-design/canvas-fonts/DMMono-Regular.ttf +0 -0
  243. package/skills/skills/canvas-design/canvas-fonts/EricaOne-OFL.txt +94 -0
  244. package/skills/skills/canvas-design/canvas-fonts/EricaOne-Regular.ttf +0 -0
  245. package/skills/skills/canvas-design/canvas-fonts/GeistMono-Bold.ttf +0 -0
  246. package/skills/skills/canvas-design/canvas-fonts/GeistMono-OFL.txt +93 -0
  247. package/skills/skills/canvas-design/canvas-fonts/GeistMono-Regular.ttf +0 -0
  248. package/skills/skills/canvas-design/canvas-fonts/Gloock-OFL.txt +93 -0
  249. package/skills/skills/canvas-design/canvas-fonts/Gloock-Regular.ttf +0 -0
  250. package/skills/skills/canvas-design/canvas-fonts/IBMPlexMono-Bold.ttf +0 -0
  251. package/skills/skills/canvas-design/canvas-fonts/IBMPlexMono-OFL.txt +93 -0
  252. package/skills/skills/canvas-design/canvas-fonts/IBMPlexMono-Regular.ttf +0 -0
  253. package/skills/skills/canvas-design/canvas-fonts/IBMPlexSerif-Bold.ttf +0 -0
  254. package/skills/skills/canvas-design/canvas-fonts/IBMPlexSerif-BoldItalic.ttf +0 -0
  255. package/skills/skills/canvas-design/canvas-fonts/IBMPlexSerif-Italic.ttf +0 -0
  256. package/skills/skills/canvas-design/canvas-fonts/IBMPlexSerif-Regular.ttf +0 -0
  257. package/skills/skills/canvas-design/canvas-fonts/InstrumentSans-Bold.ttf +0 -0
  258. package/skills/skills/canvas-design/canvas-fonts/InstrumentSans-BoldItalic.ttf +0 -0
  259. package/skills/skills/canvas-design/canvas-fonts/InstrumentSans-Italic.ttf +0 -0
  260. package/skills/skills/canvas-design/canvas-fonts/InstrumentSans-OFL.txt +93 -0
  261. package/skills/skills/canvas-design/canvas-fonts/InstrumentSans-Regular.ttf +0 -0
  262. package/skills/skills/canvas-design/canvas-fonts/InstrumentSerif-Italic.ttf +0 -0
  263. package/skills/skills/canvas-design/canvas-fonts/InstrumentSerif-Regular.ttf +0 -0
  264. package/skills/skills/canvas-design/canvas-fonts/Italiana-OFL.txt +93 -0
  265. package/skills/skills/canvas-design/canvas-fonts/Italiana-Regular.ttf +0 -0
  266. package/skills/skills/canvas-design/canvas-fonts/JetBrainsMono-Bold.ttf +0 -0
  267. package/skills/skills/canvas-design/canvas-fonts/JetBrainsMono-OFL.txt +93 -0
  268. package/skills/skills/canvas-design/canvas-fonts/JetBrainsMono-Regular.ttf +0 -0
  269. package/skills/skills/canvas-design/canvas-fonts/Jura-Light.ttf +0 -0
  270. package/skills/skills/canvas-design/canvas-fonts/Jura-Medium.ttf +0 -0
  271. package/skills/skills/canvas-design/canvas-fonts/Jura-OFL.txt +93 -0
  272. package/skills/skills/canvas-design/canvas-fonts/LibreBaskerville-OFL.txt +93 -0
  273. package/skills/skills/canvas-design/canvas-fonts/LibreBaskerville-Regular.ttf +0 -0
  274. package/skills/skills/canvas-design/canvas-fonts/Lora-Bold.ttf +0 -0
  275. package/skills/skills/canvas-design/canvas-fonts/Lora-BoldItalic.ttf +0 -0
  276. package/skills/skills/canvas-design/canvas-fonts/Lora-Italic.ttf +0 -0
  277. package/skills/skills/canvas-design/canvas-fonts/Lora-OFL.txt +93 -0
  278. package/skills/skills/canvas-design/canvas-fonts/Lora-Regular.ttf +0 -0
  279. package/skills/skills/canvas-design/canvas-fonts/NationalPark-Bold.ttf +0 -0
  280. package/skills/skills/canvas-design/canvas-fonts/NationalPark-OFL.txt +93 -0
  281. package/skills/skills/canvas-design/canvas-fonts/NationalPark-Regular.ttf +0 -0
  282. package/skills/skills/canvas-design/canvas-fonts/NothingYouCouldDo-OFL.txt +93 -0
  283. package/skills/skills/canvas-design/canvas-fonts/NothingYouCouldDo-Regular.ttf +0 -0
  284. package/skills/skills/canvas-design/canvas-fonts/Outfit-Bold.ttf +0 -0
  285. package/skills/skills/canvas-design/canvas-fonts/Outfit-OFL.txt +93 -0
  286. package/skills/skills/canvas-design/canvas-fonts/Outfit-Regular.ttf +0 -0
  287. package/skills/skills/canvas-design/canvas-fonts/PixelifySans-Medium.ttf +0 -0
  288. package/skills/skills/canvas-design/canvas-fonts/PixelifySans-OFL.txt +93 -0
  289. package/skills/skills/canvas-design/canvas-fonts/PoiretOne-OFL.txt +93 -0
  290. package/skills/skills/canvas-design/canvas-fonts/PoiretOne-Regular.ttf +0 -0
  291. package/skills/skills/canvas-design/canvas-fonts/RedHatMono-Bold.ttf +0 -0
  292. package/skills/skills/canvas-design/canvas-fonts/RedHatMono-OFL.txt +93 -0
  293. package/skills/skills/canvas-design/canvas-fonts/RedHatMono-Regular.ttf +0 -0
  294. package/skills/skills/canvas-design/canvas-fonts/Silkscreen-OFL.txt +93 -0
  295. package/skills/skills/canvas-design/canvas-fonts/Silkscreen-Regular.ttf +0 -0
  296. package/skills/skills/canvas-design/canvas-fonts/SmoochSans-Medium.ttf +0 -0
  297. package/skills/skills/canvas-design/canvas-fonts/SmoochSans-OFL.txt +93 -0
  298. package/skills/skills/canvas-design/canvas-fonts/Tektur-Medium.ttf +0 -0
  299. package/skills/skills/canvas-design/canvas-fonts/Tektur-OFL.txt +93 -0
  300. package/skills/skills/canvas-design/canvas-fonts/Tektur-Regular.ttf +0 -0
  301. package/skills/skills/canvas-design/canvas-fonts/WorkSans-Bold.ttf +0 -0
  302. package/skills/skills/canvas-design/canvas-fonts/WorkSans-BoldItalic.ttf +0 -0
  303. package/skills/skills/canvas-design/canvas-fonts/WorkSans-Italic.ttf +0 -0
  304. package/skills/skills/canvas-design/canvas-fonts/WorkSans-OFL.txt +93 -0
  305. package/skills/skills/canvas-design/canvas-fonts/WorkSans-Regular.ttf +0 -0
  306. package/skills/skills/canvas-design/canvas-fonts/YoungSerif-OFL.txt +93 -0
  307. package/skills/skills/canvas-design/canvas-fonts/YoungSerif-Regular.ttf +0 -0
  308. package/skills/skills/doc-coauthoring/SKILL.md +375 -0
  309. package/skills/skills/docx/LICENSE.txt +30 -0
  310. package/skills/skills/docx/SKILL.md +197 -0
  311. package/skills/skills/docx/docx-js.md +350 -0
  312. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-chart.xsd +1499 -0
  313. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-chartDrawing.xsd +146 -0
  314. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-diagram.xsd +1085 -0
  315. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-lockedCanvas.xsd +11 -0
  316. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-main.xsd +3081 -0
  317. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-picture.xsd +23 -0
  318. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-spreadsheetDrawing.xsd +185 -0
  319. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-wordprocessingDrawing.xsd +287 -0
  320. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/pml.xsd +1676 -0
  321. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-additionalCharacteristics.xsd +28 -0
  322. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-bibliography.xsd +144 -0
  323. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-commonSimpleTypes.xsd +174 -0
  324. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-customXmlDataProperties.xsd +25 -0
  325. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-customXmlSchemaProperties.xsd +18 -0
  326. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesCustom.xsd +59 -0
  327. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesExtended.xsd +56 -0
  328. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesVariantTypes.xsd +195 -0
  329. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-math.xsd +582 -0
  330. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-relationshipReference.xsd +25 -0
  331. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/sml.xsd +4439 -0
  332. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-main.xsd +570 -0
  333. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-officeDrawing.xsd +509 -0
  334. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-presentationDrawing.xsd +12 -0
  335. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-spreadsheetDrawing.xsd +108 -0
  336. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-wordprocessingDrawing.xsd +96 -0
  337. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/wml.xsd +3646 -0
  338. package/skills/skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/xml.xsd +116 -0
  339. package/skills/skills/docx/ooxml/schemas/ecma/fouth-edition/opc-contentTypes.xsd +42 -0
  340. package/skills/skills/docx/ooxml/schemas/ecma/fouth-edition/opc-coreProperties.xsd +50 -0
  341. package/skills/skills/docx/ooxml/schemas/ecma/fouth-edition/opc-digSig.xsd +49 -0
  342. package/skills/skills/docx/ooxml/schemas/ecma/fouth-edition/opc-relationships.xsd +33 -0
  343. package/skills/skills/docx/ooxml/schemas/mce/mc.xsd +75 -0
  344. package/skills/skills/docx/ooxml/schemas/microsoft/wml-2010.xsd +560 -0
  345. package/skills/skills/docx/ooxml/schemas/microsoft/wml-2012.xsd +67 -0
  346. package/skills/skills/docx/ooxml/schemas/microsoft/wml-2018.xsd +14 -0
  347. package/skills/skills/docx/ooxml/schemas/microsoft/wml-cex-2018.xsd +20 -0
  348. package/skills/skills/docx/ooxml/schemas/microsoft/wml-cid-2016.xsd +13 -0
  349. package/skills/skills/docx/ooxml/schemas/microsoft/wml-sdtdatahash-2020.xsd +4 -0
  350. package/skills/skills/docx/ooxml/schemas/microsoft/wml-symex-2015.xsd +8 -0
  351. package/skills/skills/docx/ooxml/scripts/pack.py +159 -0
  352. package/skills/skills/docx/ooxml/scripts/unpack.py +29 -0
  353. package/skills/skills/docx/ooxml/scripts/validate.py +69 -0
  354. package/skills/skills/docx/ooxml/scripts/validation/__init__.py +15 -0
  355. package/skills/skills/docx/ooxml/scripts/validation/base.py +951 -0
  356. package/skills/skills/docx/ooxml/scripts/validation/docx.py +274 -0
  357. package/skills/skills/docx/ooxml/scripts/validation/pptx.py +315 -0
  358. package/skills/skills/docx/ooxml/scripts/validation/redlining.py +279 -0
  359. package/skills/skills/docx/ooxml.md +610 -0
  360. package/skills/skills/docx/scripts/__init__.py +1 -0
  361. package/skills/skills/docx/scripts/document.py +1276 -0
  362. package/skills/skills/docx/scripts/templates/comments.xml +3 -0
  363. package/skills/skills/docx/scripts/templates/commentsExtended.xml +3 -0
  364. package/skills/skills/docx/scripts/templates/commentsExtensible.xml +3 -0
  365. package/skills/skills/docx/scripts/templates/commentsIds.xml +3 -0
  366. package/skills/skills/docx/scripts/templates/people.xml +3 -0
  367. package/skills/skills/docx/scripts/utilities.py +374 -0
  368. package/skills/skills/frontend-design/LICENSE.txt +177 -0
  369. package/skills/skills/frontend-design/SKILL.md +42 -0
  370. package/skills/skills/internal-comms/LICENSE.txt +202 -0
  371. package/skills/skills/internal-comms/SKILL.md +32 -0
  372. package/skills/skills/internal-comms/examples/3p-updates.md +47 -0
  373. package/skills/skills/internal-comms/examples/company-newsletter.md +65 -0
  374. package/skills/skills/internal-comms/examples/faq-answers.md +30 -0
  375. package/skills/skills/internal-comms/examples/general-comms.md +16 -0
  376. package/skills/skills/mcp-builder/LICENSE.txt +202 -0
  377. package/skills/skills/mcp-builder/SKILL.md +236 -0
  378. package/skills/skills/mcp-builder/reference/evaluation.md +602 -0
  379. package/skills/skills/mcp-builder/reference/mcp_best_practices.md +249 -0
  380. package/skills/skills/mcp-builder/reference/node_mcp_server.md +970 -0
  381. package/skills/skills/mcp-builder/reference/python_mcp_server.md +719 -0
  382. package/skills/skills/mcp-builder/scripts/connections.py +151 -0
  383. package/skills/skills/mcp-builder/scripts/evaluation.py +373 -0
  384. package/skills/skills/mcp-builder/scripts/example_evaluation.xml +22 -0
  385. package/skills/skills/mcp-builder/scripts/requirements.txt +2 -0
  386. package/skills/skills/pdf/LICENSE.txt +30 -0
  387. package/skills/skills/pdf/SKILL.md +294 -0
  388. package/skills/skills/pdf/forms.md +205 -0
  389. package/skills/skills/pdf/reference.md +612 -0
  390. package/skills/skills/pdf/scripts/check_bounding_boxes.py +70 -0
  391. package/skills/skills/pdf/scripts/check_bounding_boxes_test.py +226 -0
  392. package/skills/skills/pdf/scripts/check_fillable_fields.py +12 -0
  393. package/skills/skills/pdf/scripts/convert_pdf_to_images.py +35 -0
  394. package/skills/skills/pdf/scripts/create_validation_image.py +41 -0
  395. package/skills/skills/pdf/scripts/extract_form_field_info.py +152 -0
  396. package/skills/skills/pdf/scripts/fill_fillable_fields.py +114 -0
  397. package/skills/skills/pdf/scripts/fill_pdf_form_with_annotations.py +108 -0
  398. package/skills/skills/pptx/LICENSE.txt +30 -0
  399. package/skills/skills/pptx/SKILL.md +484 -0
  400. package/skills/skills/pptx/html2pptx.md +625 -0
  401. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-chart.xsd +1499 -0
  402. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-chartDrawing.xsd +146 -0
  403. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-diagram.xsd +1085 -0
  404. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-lockedCanvas.xsd +11 -0
  405. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-main.xsd +3081 -0
  406. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-picture.xsd +23 -0
  407. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-spreadsheetDrawing.xsd +185 -0
  408. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-wordprocessingDrawing.xsd +287 -0
  409. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/pml.xsd +1676 -0
  410. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-additionalCharacteristics.xsd +28 -0
  411. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-bibliography.xsd +144 -0
  412. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-commonSimpleTypes.xsd +174 -0
  413. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-customXmlDataProperties.xsd +25 -0
  414. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-customXmlSchemaProperties.xsd +18 -0
  415. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesCustom.xsd +59 -0
  416. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesExtended.xsd +56 -0
  417. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesVariantTypes.xsd +195 -0
  418. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-math.xsd +582 -0
  419. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-relationshipReference.xsd +25 -0
  420. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/sml.xsd +4439 -0
  421. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/vml-main.xsd +570 -0
  422. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/vml-officeDrawing.xsd +509 -0
  423. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/vml-presentationDrawing.xsd +12 -0
  424. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/vml-spreadsheetDrawing.xsd +108 -0
  425. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/vml-wordprocessingDrawing.xsd +96 -0
  426. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/wml.xsd +3646 -0
  427. package/skills/skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/xml.xsd +116 -0
  428. package/skills/skills/pptx/ooxml/schemas/ecma/fouth-edition/opc-contentTypes.xsd +42 -0
  429. package/skills/skills/pptx/ooxml/schemas/ecma/fouth-edition/opc-coreProperties.xsd +50 -0
  430. package/skills/skills/pptx/ooxml/schemas/ecma/fouth-edition/opc-digSig.xsd +49 -0
  431. package/skills/skills/pptx/ooxml/schemas/ecma/fouth-edition/opc-relationships.xsd +33 -0
  432. package/skills/skills/pptx/ooxml/schemas/mce/mc.xsd +75 -0
  433. package/skills/skills/pptx/ooxml/schemas/microsoft/wml-2010.xsd +560 -0
  434. package/skills/skills/pptx/ooxml/schemas/microsoft/wml-2012.xsd +67 -0
  435. package/skills/skills/pptx/ooxml/schemas/microsoft/wml-2018.xsd +14 -0
  436. package/skills/skills/pptx/ooxml/schemas/microsoft/wml-cex-2018.xsd +20 -0
  437. package/skills/skills/pptx/ooxml/schemas/microsoft/wml-cid-2016.xsd +13 -0
  438. package/skills/skills/pptx/ooxml/schemas/microsoft/wml-sdtdatahash-2020.xsd +4 -0
  439. package/skills/skills/pptx/ooxml/schemas/microsoft/wml-symex-2015.xsd +8 -0
  440. package/skills/skills/pptx/ooxml/scripts/pack.py +159 -0
  441. package/skills/skills/pptx/ooxml/scripts/unpack.py +29 -0
  442. package/skills/skills/pptx/ooxml/scripts/validate.py +69 -0
  443. package/skills/skills/pptx/ooxml/scripts/validation/__init__.py +15 -0
  444. package/skills/skills/pptx/ooxml/scripts/validation/base.py +951 -0
  445. package/skills/skills/pptx/ooxml/scripts/validation/docx.py +274 -0
  446. package/skills/skills/pptx/ooxml/scripts/validation/pptx.py +315 -0
  447. package/skills/skills/pptx/ooxml/scripts/validation/redlining.py +279 -0
  448. package/skills/skills/pptx/ooxml.md +427 -0
  449. package/skills/skills/pptx/scripts/html2pptx.js +979 -0
  450. package/skills/skills/pptx/scripts/inventory.py +1020 -0
  451. package/skills/skills/pptx/scripts/rearrange.py +231 -0
  452. package/skills/skills/pptx/scripts/replace.py +385 -0
  453. package/skills/skills/pptx/scripts/thumbnail.py +450 -0
  454. package/skills/skills/skill-creator/LICENSE.txt +202 -0
  455. package/skills/skills/skill-creator/SKILL.md +356 -0
  456. package/skills/skills/skill-creator/references/output-patterns.md +82 -0
  457. package/skills/skills/skill-creator/references/workflows.md +28 -0
  458. package/skills/skills/skill-creator/scripts/init_skill.py +303 -0
  459. package/skills/skills/skill-creator/scripts/package_skill.py +110 -0
  460. package/skills/skills/skill-creator/scripts/quick_validate.py +95 -0
  461. package/skills/skills/slack-gif-creator/LICENSE.txt +202 -0
  462. package/skills/skills/slack-gif-creator/SKILL.md +254 -0
  463. package/skills/skills/slack-gif-creator/core/easing.py +234 -0
  464. package/skills/skills/slack-gif-creator/core/frame_composer.py +176 -0
  465. package/skills/skills/slack-gif-creator/core/gif_builder.py +269 -0
  466. package/skills/skills/slack-gif-creator/core/validators.py +136 -0
  467. package/skills/skills/slack-gif-creator/requirements.txt +4 -0
  468. package/skills/skills/theme-factory/LICENSE.txt +202 -0
  469. package/skills/skills/theme-factory/SKILL.md +59 -0
  470. package/skills/skills/theme-factory/theme-showcase.pdf +0 -0
  471. package/skills/skills/theme-factory/themes/arctic-frost.md +19 -0
  472. package/skills/skills/theme-factory/themes/botanical-garden.md +19 -0
  473. package/skills/skills/theme-factory/themes/desert-rose.md +19 -0
  474. package/skills/skills/theme-factory/themes/forest-canopy.md +19 -0
  475. package/skills/skills/theme-factory/themes/golden-hour.md +19 -0
  476. package/skills/skills/theme-factory/themes/midnight-galaxy.md +19 -0
  477. package/skills/skills/theme-factory/themes/modern-minimalist.md +19 -0
  478. package/skills/skills/theme-factory/themes/ocean-depths.md +19 -0
  479. package/skills/skills/theme-factory/themes/sunset-boulevard.md +19 -0
  480. package/skills/skills/theme-factory/themes/tech-innovation.md +19 -0
  481. package/skills/skills/web-artifacts-builder/LICENSE.txt +202 -0
  482. package/skills/skills/web-artifacts-builder/SKILL.md +74 -0
  483. package/skills/skills/web-artifacts-builder/scripts/bundle-artifact.sh +54 -0
  484. package/skills/skills/web-artifacts-builder/scripts/init-artifact.sh +322 -0
  485. package/skills/skills/webapp-testing/LICENSE.txt +202 -0
  486. package/skills/skills/webapp-testing/SKILL.md +96 -0
  487. package/skills/skills/webapp-testing/examples/console_logging.py +35 -0
  488. package/skills/skills/webapp-testing/examples/element_discovery.py +40 -0
  489. package/skills/skills/webapp-testing/examples/static_html_automation.py +33 -0
  490. package/skills/skills/webapp-testing/scripts/with_server.py +106 -0
  491. package/skills/skills/xlsx/LICENSE.txt +30 -0
  492. package/skills/skills/xlsx/SKILL.md +289 -0
  493. package/skills/skills/xlsx/recalc.py +178 -0
  494. package/skills/spec/agent-skills-spec.md +3 -0
  495. package/skills/template/SKILL.md +6 -0
  496. package/src/agents.ts +504 -0
  497. package/src/ai-client.ts +1456 -0
  498. package/src/auth.ts +648 -0
  499. package/src/cancellation.ts +176 -0
  500. package/src/checkpoint.ts +219 -0
  501. package/src/cli.ts +384 -0
  502. package/src/config.ts +248 -0
  503. package/src/context-compressor.ts +290 -0
  504. package/src/conversation.ts +288 -0
  505. package/src/gui-subagent/action-parser/actionParser.ts +312 -0
  506. package/src/gui-subagent/action-parser/constants.ts +12 -0
  507. package/src/gui-subagent/action-parser/index.ts +6 -0
  508. package/src/gui-subagent/action-parser/types.ts +31 -0
  509. package/src/gui-subagent/agent/gui-agent.ts +982 -0
  510. package/src/gui-subagent/agent/index.ts +5 -0
  511. package/src/gui-subagent/index.ts +139 -0
  512. package/src/gui-subagent/operator/base-operator.ts +246 -0
  513. package/src/gui-subagent/operator/computer-operator.ts +520 -0
  514. package/src/gui-subagent/operator/index.ts +7 -0
  515. package/src/gui-subagent/types/actions.ts +263 -0
  516. package/src/gui-subagent/types/index.ts +6 -0
  517. package/src/gui-subagent/types/operator.ts +106 -0
  518. package/src/gui-subagent/utils.ts +51 -0
  519. package/src/index.ts +18 -0
  520. package/src/input-processor.ts +282 -0
  521. package/src/logger.ts +438 -0
  522. package/src/mcp.ts +563 -0
  523. package/src/memory.ts +303 -0
  524. package/src/session-manager.ts +308 -0
  525. package/src/session.ts +1280 -0
  526. package/src/skill-invoker.ts +1888 -0
  527. package/src/skill-loader.ts +476 -0
  528. package/src/slash-commands.ts +1150 -0
  529. package/src/smart-approval.ts +595 -0
  530. package/src/system-prompt-generator.ts +786 -0
  531. package/src/theme.ts +455 -0
  532. package/src/tools.ts +3398 -0
  533. package/src/types.ts +198 -0
  534. package/src/update.ts +270 -0
  535. package/src/workflow.ts +508 -0
  536. package/tsconfig.json +22 -0
  537. package/vitest.config.ts +19 -0
@@ -0,0 +1,441 @@
1
+ /**
2
+ * Computer Operator using @computer-use/nut-js
3
+ * Provides desktop automation capabilities for gui-subagent
4
+ * Based on UI-TARS NutJSOperator implementation
5
+ *
6
+ * This implementation is aligned with packages/ui-tars/operators/nut-js/src/index.ts
7
+ */
8
+ import { screen, Button, Key, Point, keyboard, mouse, sleep, straightTo, clipboard, } from '@computer-use/nut-js';
9
+ import { Jimp } from 'jimp';
10
+ import { Operator, parseBoxToScreenCoords } from './base-operator.js';
11
+ import { getLogger } from '../../logger.js';
12
+ const guiLogger = getLogger();
13
+ export class ComputerOperator extends Operator {
14
+ config;
15
+ logger;
16
+ screenCtx = null;
17
+ constructor(options = {}) {
18
+ super();
19
+ this.config = options.config || {};
20
+ this.logger = options.logger || guiLogger;
21
+ }
22
+ async initialize() {
23
+ this.logger.debug('Initializing computer operator...');
24
+ try {
25
+ const { width, height, scaleFactor } = await this.getScreenSize();
26
+ this.screenCtx = {
27
+ width,
28
+ height,
29
+ scaleFactor,
30
+ };
31
+ this.logger.debug(`Computer operator initialized: ${width}x${height} @ ${scaleFactor}x`);
32
+ }
33
+ catch (error) {
34
+ this.logger.error('Failed to initialize computer operator:', error);
35
+ throw error;
36
+ }
37
+ }
38
+ async getScreenSize() {
39
+ try {
40
+ const grabImage = await screen.grab();
41
+ const screenWithScale = await grabImage.toRGB();
42
+ const scaleFactor = screenWithScale.pixelDensity.scaleX;
43
+ const width = screenWithScale.width / scaleFactor;
44
+ const height = screenWithScale.height / scaleFactor;
45
+ return { width, height, scaleFactor };
46
+ }
47
+ catch {
48
+ return {
49
+ width: this.config.viewport?.width || 1920,
50
+ height: this.config.viewport?.height || 1080,
51
+ scaleFactor: this.config.deviceScaleFactor || 1,
52
+ };
53
+ }
54
+ }
55
+ getSupportedActions() {
56
+ return [
57
+ 'click',
58
+ 'left_click',
59
+ 'left_single',
60
+ 'left_double',
61
+ 'double_click',
62
+ 'right_click',
63
+ 'right_single',
64
+ 'middle_click',
65
+ 'mouse_move',
66
+ 'hover',
67
+ 'drag',
68
+ 'left_click_drag',
69
+ 'select',
70
+ 'scroll',
71
+ 'type',
72
+ 'hotkey',
73
+ 'press',
74
+ 'release',
75
+ 'open_url',
76
+ 'wait',
77
+ 'finished',
78
+ 'user_stop',
79
+ 'error_env',
80
+ 'call_user',
81
+ ];
82
+ }
83
+ screenContext() {
84
+ if (!this.screenCtx) {
85
+ throw new Error('Screen context not initialized');
86
+ }
87
+ return this.screenCtx;
88
+ }
89
+ async screenshot() {
90
+ try {
91
+ const grabImage = await screen.grab();
92
+ const screenWithScale = await grabImage.toRGB();
93
+ const scaleFactor = screenWithScale.pixelDensity.scaleX;
94
+ const screenWithScaleImage = await Jimp.fromBitmap({
95
+ width: screenWithScale.width,
96
+ height: screenWithScale.height,
97
+ data: Buffer.from(screenWithScale.data),
98
+ });
99
+ const width = screenWithScale.width / scaleFactor;
100
+ const height = screenWithScale.height / scaleFactor;
101
+ const physicalScreenImage = await screenWithScaleImage
102
+ .resize({
103
+ w: width,
104
+ h: height,
105
+ })
106
+ .getBuffer('image/png');
107
+ this.logger.debug(`[ComputerOperator] screenshot: ${width}x${height}, scaleFactor: ${scaleFactor}`);
108
+ return {
109
+ status: 'success',
110
+ base64: physicalScreenImage.toString('base64'),
111
+ scaleFactor,
112
+ };
113
+ }
114
+ catch (error) {
115
+ const errorMsg = error instanceof Error ? error.message : 'Unknown error';
116
+ this.logger.warn(`[ComputerOperator] Screenshot failed: ${errorMsg}`);
117
+ return {
118
+ status: 'failed',
119
+ errorMessage: errorMsg,
120
+ };
121
+ }
122
+ }
123
+ async execute(params) {
124
+ const { parsedPrediction, screenWidth, screenHeight, scaleFactor } = params;
125
+ const { action_type, action_inputs } = parsedPrediction;
126
+ // Empty or invalid action should return failed to avoid infinite loop
127
+ if (!action_type || action_type.trim() === '') {
128
+ this.logger.warn(`[ComputerOperator] Empty action, skipping step`);
129
+ return {
130
+ status: 'failed',
131
+ errorMessage: 'Empty or invalid action type'
132
+ };
133
+ }
134
+ const startBoxStr = action_inputs?.start_box || '';
135
+ const { x: startX, y: startY } = parseBoxToScreenCoords({
136
+ boxStr: startBoxStr,
137
+ screenWidth,
138
+ screenHeight,
139
+ });
140
+ mouse.config.mouseSpeed = 3600;
141
+ // this.logger.debug('[ComputerOperator] execute', { action_type, startX, startY, scaleFactor });
142
+ try {
143
+ const result = await this.executeAction(action_type, action_inputs, { startX, startY, screenWidth, screenHeight, scaleFactor });
144
+ if (result === 'end') {
145
+ return { status: 'end' };
146
+ }
147
+ return { status: 'success' };
148
+ }
149
+ catch (error) {
150
+ this.logger.error(`Failed to execute action ${action_type}:`, error);
151
+ return {
152
+ status: 'failed',
153
+ errorMessage: error.message,
154
+ };
155
+ }
156
+ }
157
+ async executeAction(actionType, inputs, context) {
158
+ const { startX, startY, screenWidth, screenHeight, scaleFactor } = context;
159
+ const moveStraightTo = async (x, y) => {
160
+ await mouse.move(straightTo(new Point(x, y)));
161
+ };
162
+ const getHotkeys = (keyStr) => {
163
+ if (keyStr) {
164
+ const platformCommandKey = process.platform === 'darwin' ? Key.LeftCmd : Key.LeftWin;
165
+ const platformCtrlKey = process.platform === 'darwin' ? Key.LeftCmd : Key.LeftControl;
166
+ const keyMap = {
167
+ return: Key.Enter,
168
+ ctrl: platformCtrlKey,
169
+ shift: Key.LeftShift,
170
+ alt: Key.LeftAlt,
171
+ 'page down': Key.PageDown,
172
+ 'page up': Key.PageUp,
173
+ meta: platformCommandKey,
174
+ win: platformCommandKey,
175
+ command: platformCommandKey,
176
+ cmd: platformCommandKey,
177
+ ',': Key.Comma,
178
+ arrowup: Key.Up,
179
+ arrowdown: Key.Down,
180
+ arrowleft: Key.Left,
181
+ arrowright: Key.Right,
182
+ };
183
+ const lowercaseKeyMap = Object.fromEntries(Object.entries(Key).map(([k, v]) => [k.toLowerCase(), v]));
184
+ const keys = keyStr
185
+ .split(/[\s+]+/)
186
+ .map((k) => k.toLowerCase())
187
+ .map((k) => keyMap[k] ??
188
+ lowercaseKeyMap[k])
189
+ .filter(Boolean);
190
+ this.logger.debug('[ComputerOperator] hotkey:', keys);
191
+ return keys;
192
+ }
193
+ return [];
194
+ };
195
+ switch (actionType) {
196
+ case 'wait':
197
+ this.logger.debug('[ComputerOperator] wait', inputs);
198
+ await sleep(5000);
199
+ break;
200
+ case 'mouse_move':
201
+ case 'hover':
202
+ this.logger.debug('[ComputerOperator] mouse_move');
203
+ await moveStraightTo(startX, startY);
204
+ break;
205
+ case 'click':
206
+ case 'left_click':
207
+ case 'left_single':
208
+ this.logger.debug('[ComputerOperator] left_click');
209
+ await moveStraightTo(startX, startY);
210
+ await sleep(100);
211
+ await mouse.click(Button.LEFT);
212
+ break;
213
+ case 'left_double':
214
+ case 'double_click':
215
+ this.logger.debug(`[ComputerOperator] ${actionType}(${startX}, ${startY})`);
216
+ await moveStraightTo(startX, startY);
217
+ await sleep(100);
218
+ await mouse.doubleClick(Button.LEFT);
219
+ break;
220
+ case 'right_click':
221
+ case 'right_single':
222
+ this.logger.debug('[ComputerOperator] right_click');
223
+ await moveStraightTo(startX, startY);
224
+ await sleep(100);
225
+ await mouse.click(Button.RIGHT);
226
+ break;
227
+ case 'middle_click':
228
+ this.logger.debug('[ComputerOperator] middle_click');
229
+ await moveStraightTo(startX, startY);
230
+ await mouse.click(Button.MIDDLE);
231
+ break;
232
+ case 'drag':
233
+ case 'left_click_drag':
234
+ case 'select': {
235
+ const endBoxStr = inputs?.end_box || '';
236
+ if (endBoxStr) {
237
+ const { x: endX, y: endY } = parseBoxToScreenCoords({
238
+ boxStr: endBoxStr,
239
+ screenWidth,
240
+ screenHeight,
241
+ });
242
+ if (startX && startY && endX && endY) {
243
+ this.logger.debug(`[ComputerOperator] drag coordinates: startX=${startX}, startY=${startY}, endX=${endX}, endY=${endY}`);
244
+ await moveStraightTo(startX, startY);
245
+ await sleep(100);
246
+ await mouse.drag(straightTo(new Point(endX, endY)));
247
+ }
248
+ }
249
+ break;
250
+ }
251
+ case 'type': {
252
+ const content = inputs.content?.trim();
253
+ this.logger.debug('[ComputerOperator] type', content);
254
+ if (content) {
255
+ const stripContent = content.replace(/\\n$/, '').replace(/\n$/, '');
256
+ keyboard.config.autoDelayMs = 0;
257
+ if (process.platform === 'win32') {
258
+ const originalClipboard = await clipboard.getContent();
259
+ await clipboard.setContent(stripContent);
260
+ await keyboard.pressKey(Key.LeftControl, Key.V);
261
+ await sleep(50);
262
+ await keyboard.releaseKey(Key.LeftControl, Key.V);
263
+ await sleep(50);
264
+ await clipboard.setContent(originalClipboard);
265
+ }
266
+ else {
267
+ await keyboard.type(stripContent);
268
+ }
269
+ if (content.endsWith('\n') || content.endsWith('\\n')) {
270
+ await keyboard.pressKey(Key.Enter);
271
+ await keyboard.releaseKey(Key.Enter);
272
+ }
273
+ keyboard.config.autoDelayMs = 500;
274
+ }
275
+ break;
276
+ }
277
+ case 'hotkey': {
278
+ const keyStr = inputs?.key || inputs?.hotkey;
279
+ const keys = getHotkeys(keyStr);
280
+ if (keys.length > 0) {
281
+ await keyboard.pressKey(...keys);
282
+ await keyboard.releaseKey(...keys);
283
+ }
284
+ break;
285
+ }
286
+ case 'press': {
287
+ const keyStr = inputs?.key || inputs?.hotkey;
288
+ const keys = getHotkeys(keyStr);
289
+ if (keys.length > 0) {
290
+ await keyboard.pressKey(...keys);
291
+ }
292
+ break;
293
+ }
294
+ case 'release': {
295
+ const keyStr = inputs?.key || inputs?.hotkey;
296
+ const keys = getHotkeys(keyStr);
297
+ if (keys.length > 0) {
298
+ await keyboard.releaseKey(...keys);
299
+ }
300
+ break;
301
+ }
302
+ case 'scroll': {
303
+ const { direction } = inputs;
304
+ if (startX !== null && startY !== null) {
305
+ await moveStraightTo(startX, startY);
306
+ }
307
+ switch (direction?.toLowerCase()) {
308
+ case 'up':
309
+ await mouse.scrollUp(5 * 100);
310
+ break;
311
+ case 'down':
312
+ await mouse.scrollDown(5 * 100);
313
+ break;
314
+ default:
315
+ this.logger.warn(`[ComputerOperator] Unsupported scroll direction: ${direction}`);
316
+ }
317
+ break;
318
+ }
319
+ case 'open_url': {
320
+ let url = inputs?.url || inputs?.content;
321
+ if (!url) {
322
+ throw new Error('No URL specified for open_url action');
323
+ }
324
+ // Ensure URL has protocol
325
+ if (!/^https?:\/\//i.test(url)) {
326
+ url = 'https://' + url;
327
+ }
328
+ this.logger.debug(`[ComputerOperator] Opening URL: ${url}`);
329
+ // Use system command to open URL in default browser
330
+ const { exec } = await import('child_process');
331
+ const platform = process.platform;
332
+ if (platform === 'win32') {
333
+ // Windows: use start command
334
+ await new Promise((resolve, reject) => {
335
+ exec(`start "" "${url}"`, (error) => {
336
+ if (error) {
337
+ this.logger.warn(`[ComputerOperator] Failed to open URL with start command: ${error.message}`);
338
+ // Fallback: try using PowerShell
339
+ exec(`powershell -Command "Start-Process '${url}'"`, (psError) => {
340
+ if (psError) {
341
+ reject(psError);
342
+ }
343
+ else {
344
+ resolve();
345
+ }
346
+ });
347
+ }
348
+ else {
349
+ resolve();
350
+ }
351
+ });
352
+ });
353
+ }
354
+ else if (platform === 'darwin') {
355
+ // macOS: use open command
356
+ await new Promise((resolve, reject) => {
357
+ exec(`open "${url}"`, (error) => {
358
+ if (error) {
359
+ reject(error);
360
+ }
361
+ else {
362
+ resolve();
363
+ }
364
+ });
365
+ });
366
+ }
367
+ else {
368
+ // Linux: use xdg-open
369
+ await new Promise((resolve, reject) => {
370
+ exec(`xdg-open "${url}"`, (error) => {
371
+ if (error) {
372
+ reject(error);
373
+ }
374
+ else {
375
+ resolve();
376
+ }
377
+ });
378
+ });
379
+ }
380
+ // Wait for browser to open and page to load
381
+ await sleep(2000);
382
+ break;
383
+ }
384
+ case 'error_env':
385
+ case 'call_user':
386
+ case 'finished':
387
+ case 'user_stop':
388
+ this.logger.debug(`[ComputerOperator] ${actionType}`);
389
+ return 'end';
390
+ default:
391
+ this.logger.warn(`[ComputerOperator] Unsupported action: ${actionType}`);
392
+ }
393
+ }
394
+ async cleanup() {
395
+ this.logger.debug('Cleaning up computer operator...');
396
+ }
397
+ async destroyInstance() {
398
+ this.logger.debug('Destroying computer operator instance...');
399
+ await this.cleanup();
400
+ }
401
+ static get MANUAL() {
402
+ return {
403
+ ACTION_SPACES: [
404
+ // Mouse actions
405
+ `click(start_box='[x1, y1, x2, y2]') # Single click (taskbar icons)`,
406
+ `left_double(start_box='[x1, y1, x2, y2]') # Double click (desktop icons/folders)`,
407
+ `right_single(start_box='[x1, y1, x2, y2]') # Right click`,
408
+ `drag(start_box='[x1, y1, x2, y2]', end_box='[x3, y3, x4, y4]') # Drag`,
409
+ // Keyboard actions
410
+ `hotkey(key='') # e.g., 'ctrl c', 'alt tab' (max 3 keys)`,
411
+ `type(content='') # Use "\\n" at the end to submit`,
412
+ `press(key='') # Single key press: 'enter', 'esc', 'tab', 'win', etc.`,
413
+ // Navigation
414
+ `open_url(url='https://xxx') # Open URL in default browser`,
415
+ // Scroll
416
+ `scroll(start_box='[x1, y1, x2, y2]', direction='down or up or right or left')`,
417
+ // System
418
+ `wait() # Sleep 5s and take a screenshot`,
419
+ `finished() # Task completed`,
420
+ `call_user() # Need user's help`,
421
+ ],
422
+ KEY_SPACE: {
423
+ 'enter': 'Enter key',
424
+ 'esc': 'Escape key',
425
+ 'tab': 'Tab key',
426
+ 'win': 'Windows key (or Command on Mac)',
427
+ 'delete': 'Delete key',
428
+ 'backspace': 'Backspace key',
429
+ 'page up': 'Page Up',
430
+ 'page down': 'Page Down',
431
+ 'home': 'Home key',
432
+ 'end': 'End key',
433
+ 'arrow up': 'Up arrow',
434
+ 'arrow down': 'Down arrow',
435
+ 'arrow left': 'Left arrow',
436
+ 'arrow right': 'Right arrow',
437
+ },
438
+ };
439
+ }
440
+ }
441
+ //# sourceMappingURL=computer-operator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"computer-operator.js","sourceRoot":"","sources":["../../../src/gui-subagent/operator/computer-operator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,MAAM,EACN,MAAM,EACN,GAAG,EACH,KAAK,EAEL,QAAQ,EACR,KAAK,EACL,KAAK,EACL,UAAU,EACV,SAAS,GACV,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,QAAQ,EAAuB,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC3F,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,MAAM,SAAS,GAAG,SAAS,EAAE,CAAC;AAQ9B,MAAM,OAAO,gBAAiB,SAAQ,QAAQ;IACpC,MAAM,CAAiB;IACvB,MAAM,CAAM;IACZ,SAAS,GAAyB,IAAI,CAAC;IAE/C,YAAY,UAAmC,EAAE;QAC/C,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC;IAC5C,CAAC;IAES,KAAK,CAAC,UAAU;QACxB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEvD,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAClE,IAAI,CAAC,SAAS,GAAG;gBACf,KAAK;gBACL,MAAM;gBACN,WAAW;aACZ,CAAC;YAEF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kCAAkC,KAAK,IAAI,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC;QAC3F,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yCAAyC,EAAE,KAAK,CAAC,CAAC;YACpE,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,eAAe,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YAChD,MAAM,WAAW,GAAG,eAAe,CAAC,YAAY,CAAC,MAAM,CAAC;YACxD,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,GAAG,WAAW,CAAC;YAClD,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,GAAG,WAAW,CAAC;YACpD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,IAAI;gBAC1C,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,IAAI,IAAI;gBAC5C,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB,IAAI,CAAC;aAChD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,mBAAmB;QACjB,OAAO;YACL,OAAO;YACP,YAAY;YACZ,aAAa;YACb,aAAa;YACb,cAAc;YACd,aAAa;YACb,cAAc;YACd,cAAc;YACd,YAAY;YACZ,OAAO;YACP,MAAM;YACN,iBAAiB;YACjB,QAAQ;YACR,QAAQ;YACR,MAAM;YACN,QAAQ;YACR,OAAO;YACP,SAAS;YACT,UAAU;YACV,MAAM;YACN,UAAU;YACV,WAAW;YACX,WAAW;YACX,WAAW;SACZ,CAAC;IACJ,CAAC;IAES,aAAa;QACrB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAES,KAAK,CAAC,UAAU;QACxB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,eAAe,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YAChD,MAAM,WAAW,GAAG,eAAe,CAAC,YAAY,CAAC,MAAM,CAAC;YAExD,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC;gBACjD,KAAK,EAAE,eAAe,CAAC,KAAK;gBAC5B,MAAM,EAAE,eAAe,CAAC,MAAM;gBAC9B,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC;aACxC,CAAC,CAAC;YAEH,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,GAAG,WAAW,CAAC;YAClD,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,GAAG,WAAW,CAAC;YAEpD,MAAM,mBAAmB,GAAG,MAAM,oBAAoB;iBACnD,MAAM,CAAC;gBACN,CAAC,EAAE,KAAK;gBACR,CAAC,EAAE,MAAM;aACV,CAAC;iBACD,SAAS,CAAC,WAAW,CAAC,CAAC;YAE1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kCAAkC,KAAK,IAAI,MAAM,kBAAkB,WAAW,EAAE,CAAC,CAAC;YAEpG,OAAO;gBACL,MAAM,EAAE,SAAS;gBACjB,MAAM,EAAE,mBAAmB,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC9C,WAAW;aACZ,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAC1E,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,QAAQ,EAAE,CAAC,CAAC;YACtE,OAAO;gBACL,MAAM,EAAE,QAAQ;gBAChB,YAAY,EAAE,QAAQ;aACvB,CAAC;QACJ,CAAC;IACH,CAAC;IAES,KAAK,CAAC,OAAO,CAAC,MAAqB;QAC3C,MAAM,EAAE,gBAAgB,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;QAC5E,MAAM,EAAE,WAAW,EAAE,aAAa,EAAE,GAAG,gBAAgB,CAAC;QAExD,sEAAsE;QACtE,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YAC9C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;YACnE,OAAO;gBACL,MAAM,EAAE,QAAQ;gBAChB,YAAY,EAAE,8BAA8B;aAC7C,CAAC;QACJ,CAAC;QAED,MAAM,WAAW,GAAG,aAAa,EAAE,SAAS,IAAI,EAAE,CAAC;QACnD,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,sBAAsB,CAAC;YACtD,MAAM,EAAE,WAAW;YACnB,WAAW;YACX,YAAY;SACb,CAAC,CAAC;QAEH,KAAK,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;QAE/B,iGAAiG;QAEjG,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,aAAa,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC,CAAC;YAChI,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;gBACrB,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YAC3B,CAAC;YAED,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,WAAW,GAAG,EAAE,KAAK,CAAC,CAAC;YACrE,OAAO;gBACL,MAAM,EAAE,QAAQ;gBAChB,YAAY,EAAG,KAAe,CAAC,OAAO;aACvC,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,UAAkB,EAClB,MAA2B,EAC3B,OAA2G;QAE3G,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;QAE3E,MAAM,cAAc,GAAG,KAAK,EAAE,CAAS,EAAE,CAAS,EAAE,EAAE;YACpD,MAAM,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC,CAAC;QAEF,MAAM,UAAU,GAAG,CAAC,MAA0B,EAAS,EAAE;YACvD,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,kBAAkB,GAAG,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC;gBACrF,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC;gBACtF,MAAM,MAAM,GAAG;oBACb,MAAM,EAAE,GAAG,CAAC,KAAK;oBACjB,IAAI,EAAE,eAAe;oBACrB,KAAK,EAAE,GAAG,CAAC,SAAS;oBACpB,GAAG,EAAE,GAAG,CAAC,OAAO;oBAChB,WAAW,EAAE,GAAG,CAAC,QAAQ;oBACzB,SAAS,EAAE,GAAG,CAAC,MAAM;oBACrB,IAAI,EAAE,kBAAkB;oBACxB,GAAG,EAAE,kBAAkB;oBACvB,OAAO,EAAE,kBAAkB;oBAC3B,GAAG,EAAE,kBAAkB;oBACvB,GAAG,EAAE,GAAG,CAAC,KAAK;oBACd,OAAO,EAAE,GAAG,CAAC,EAAE;oBACf,SAAS,EAAE,GAAG,CAAC,IAAI;oBACnB,SAAS,EAAE,GAAG,CAAC,IAAI;oBACnB,UAAU,EAAE,GAAG,CAAC,KAAK;iBACb,CAAC;gBAEX,MAAM,eAAe,GAAG,MAAM,CAAC,WAAW,CACxC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CAG1D,CAAC;gBAEF,MAAM,IAAI,GAAG,MAAM;qBAChB,KAAK,CAAC,QAAQ,CAAC;qBACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;qBAC3B,GAAG,CACF,CAAC,CAAC,EAAE,EAAE,CACJ,MAAM,CAAC,CAAwB,CAAC;oBAChC,eAAe,CAAC,CAAgC,CAAC,CACpD;qBACA,MAAM,CAAC,OAAO,CAAC,CAAC;gBACnB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE,IAAI,CAAC,CAAC;gBACtD,OAAO,IAAI,CAAC;YACd,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;QAEF,QAAQ,UAAU,EAAE,CAAC;YACnB,KAAK,MAAM;gBACT,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,MAAM,CAAC,CAAC;gBACrD,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;gBAClB,MAAM;YAER,KAAK,YAAY,CAAC;YAClB,KAAK,OAAO;gBACV,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBACnD,MAAM,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACrC,MAAM;YAER,KAAK,OAAO,CAAC;YACb,KAAK,YAAY,CAAC;YAClB,KAAK,aAAa;gBAChB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBACnD,MAAM,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACrC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;gBACjB,MAAM,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC/B,MAAM;YAER,KAAK,aAAa,CAAC;YACnB,KAAK,cAAc;gBACjB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,UAAU,IAAI,MAAM,KAAK,MAAM,GAAG,CAAC,CAAC;gBAC5E,MAAM,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACrC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;gBACjB,MAAM,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACrC,MAAM;YAER,KAAK,aAAa,CAAC;YACnB,KAAK,cAAc;gBACjB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;gBACpD,MAAM,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACrC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;gBACjB,MAAM,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAChC,MAAM;YAER,KAAK,cAAc;gBACjB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;gBACrD,MAAM,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACrC,MAAM,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACjC,MAAM;YAER,KAAK,MAAM,CAAC;YACZ,KAAK,iBAAiB,CAAC;YACvB,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,MAAM,SAAS,GAAG,MAAM,EAAE,OAAO,IAAI,EAAE,CAAC;gBACxC,IAAI,SAAS,EAAE,CAAC;oBACd,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,GAAG,sBAAsB,CAAC;wBAClD,MAAM,EAAE,SAAS;wBACjB,WAAW;wBACX,YAAY;qBACb,CAAC,CAAC;oBAEH,IAAI,MAAM,IAAI,MAAM,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;wBACrC,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,+CAA+C,MAAM,YAAY,MAAM,UAAU,IAAI,UAAU,IAAI,EAAE,CACtG,CAAC;wBACF,MAAM,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;wBACrC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;wBACjB,MAAM,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;oBACtD,CAAC;gBACH,CAAC;gBACD,MAAM;YACR,CAAC;YAED,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;gBACvC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,OAAO,CAAC,CAAC;gBACtD,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBACpE,QAAQ,CAAC,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC;oBAChC,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;wBACjC,MAAM,iBAAiB,GAAG,MAAM,SAAS,CAAC,UAAU,EAAE,CAAC;wBACvD,MAAM,SAAS,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;wBACzC,MAAM,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;wBAChD,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;wBAChB,MAAM,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;wBAClD,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;wBAChB,MAAM,SAAS,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC;oBAChD,CAAC;yBAAM,CAAC;wBACN,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBACpC,CAAC;oBAED,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;wBACtD,MAAM,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;wBACnC,MAAM,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBACvC,CAAC;oBAED,QAAQ,CAAC,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC;gBACpC,CAAC;gBACD,MAAM;YACR,CAAC;YAED,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,MAAM,MAAM,GAAG,MAAM,EAAE,GAAG,IAAI,MAAM,EAAE,MAAM,CAAC;gBAC7C,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;gBAChC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACpB,MAAM,QAAQ,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC;oBACjC,MAAM,QAAQ,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC;gBACrC,CAAC;gBACD,MAAM;YACR,CAAC;YAED,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,MAAM,MAAM,GAAG,MAAM,EAAE,GAAG,IAAI,MAAM,EAAE,MAAM,CAAC;gBAC7C,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;gBAChC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACpB,MAAM,QAAQ,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC;gBACnC,CAAC;gBACD,MAAM;YACR,CAAC;YAED,KAAK,SAAS,CAAC,CAAC,CAAC;gBACf,MAAM,MAAM,GAAG,MAAM,EAAE,GAAG,IAAI,MAAM,EAAE,MAAM,CAAC;gBAC7C,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;gBAChC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACpB,MAAM,QAAQ,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC;gBACrC,CAAC;gBACD,MAAM;YACR,CAAC;YAED,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC;gBAC7B,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;oBACvC,MAAM,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBACvC,CAAC;gBAED,QAAQ,SAAS,EAAE,WAAW,EAAE,EAAE,CAAC;oBACjC,KAAK,IAAI;wBACP,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;wBAC9B,MAAM;oBACR,KAAK,MAAM;wBACT,MAAM,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;wBAChC,MAAM;oBACR;wBACE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oDAAoD,SAAS,EAAE,CAAC,CAAC;gBACtF,CAAC;gBACD,MAAM;YACR,CAAC;YAED,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,IAAI,GAAG,GAAG,MAAM,EAAE,GAAG,IAAI,MAAM,EAAE,OAAO,CAAC;gBACzC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;gBAC1D,CAAC;gBAED,0BAA0B;gBAC1B,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC/B,GAAG,GAAG,UAAU,GAAG,GAAG,CAAC;gBACzB,CAAC;gBAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,GAAG,EAAE,CAAC,CAAC;gBAE5D,oDAAoD;gBACpD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC;gBAC/C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;gBAElC,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;oBACzB,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;wBAC1C,IAAI,CAAC,aAAa,GAAG,GAAG,EAAE,CAAC,KAAK,EAAE,EAAE;4BAClC,IAAI,KAAK,EAAE,CAAC;gCACV,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6DAA6D,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gCAC/F,iCAAiC;gCACjC,IAAI,CAAC,uCAAuC,GAAG,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE;oCAC/D,IAAI,OAAO,EAAE,CAAC;wCACZ,MAAM,CAAC,OAAO,CAAC,CAAC;oCAClB,CAAC;yCAAM,CAAC;wCACN,OAAO,EAAE,CAAC;oCACZ,CAAC;gCACH,CAAC,CAAC,CAAC;4BACL,CAAC;iCAAM,CAAC;gCACN,OAAO,EAAE,CAAC;4BACZ,CAAC;wBACH,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC,CAAC;gBACL,CAAC;qBAAM,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;oBACjC,0BAA0B;oBAC1B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;wBAC1C,IAAI,CAAC,SAAS,GAAG,GAAG,EAAE,CAAC,KAAK,EAAE,EAAE;4BAC9B,IAAI,KAAK,EAAE,CAAC;gCACV,MAAM,CAAC,KAAK,CAAC,CAAC;4BAChB,CAAC;iCAAM,CAAC;gCACN,OAAO,EAAE,CAAC;4BACZ,CAAC;wBACH,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,sBAAsB;oBACtB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;wBAC1C,IAAI,CAAC,aAAa,GAAG,GAAG,EAAE,CAAC,KAAK,EAAE,EAAE;4BAClC,IAAI,KAAK,EAAE,CAAC;gCACV,MAAM,CAAC,KAAK,CAAC,CAAC;4BAChB,CAAC;iCAAM,CAAC;gCACN,OAAO,EAAE,CAAC;4BACZ,CAAC;wBACH,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC,CAAC;gBACL,CAAC;gBAED,4CAA4C;gBAC5C,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;gBAClB,MAAM;YACR,CAAC;YAED,KAAK,WAAW,CAAC;YACjB,KAAK,WAAW,CAAC;YACjB,KAAK,UAAU,CAAC;YAChB,KAAK,WAAW;gBACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;gBACtD,OAAO,KAAK,CAAC;YAEf;gBACE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0CAA0C,UAAU,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACvB,CAAC;IAED,MAAM,KAAc,MAAM;QACxB,OAAO;YACL,aAAa,EAAE;gBACb,gBAAgB;gBAChB,oEAAoE;gBACpE,kFAAkF;gBAClF,0DAA0D;gBAC1D,uEAAuE;gBAEvE,mBAAmB;gBACnB,yDAAyD;gBACzD,mDAAmD;gBACnD,sEAAsE;gBAEtE,aAAa;gBACb,2DAA2D;gBAE3D,SAAS;gBACT,+EAA+E;gBAE/E,SAAS;gBACT,yCAAyC;gBACzC,6BAA6B;gBAC7B,gCAAgC;aACjC;YAED,SAAS,EAAE;gBACT,OAAO,EAAE,WAAW;gBACpB,KAAK,EAAE,YAAY;gBACnB,KAAK,EAAE,SAAS;gBAChB,KAAK,EAAE,iCAAiC;gBACxC,QAAQ,EAAE,YAAY;gBACtB,WAAW,EAAE,eAAe;gBAC5B,SAAS,EAAE,SAAS;gBACpB,WAAW,EAAE,WAAW;gBACxB,MAAM,EAAE,UAAU;gBAClB,KAAK,EAAE,SAAS;gBAChB,UAAU,EAAE,UAAU;gBACtB,YAAY,EAAE,YAAY;gBAC1B,YAAY,EAAE,YAAY;gBAC1B,aAAa,EAAE,aAAa;aAC7B;SACF,CAAC;IACJ,CAAC;CACF"}
@@ -0,0 +1,55 @@
1
+ /**
2
+ * Desktop Operator - Full Desktop Automation using @computer-use/nut-js
3
+ * 参考 UI-TARS-desktop 实现
4
+ */
5
+ export interface DesktopOperatorConfig {
6
+ mouseSpeed?: number;
7
+ loopIntervalInMs?: number;
8
+ maxLoopCount?: number;
9
+ }
10
+ export interface ScreenshotOutput {
11
+ status: 'success' | 'error';
12
+ base64?: string;
13
+ scaleFactor?: number;
14
+ errorMessage?: string;
15
+ }
16
+ export interface ExecuteParams {
17
+ thought: string;
18
+ action: string;
19
+ }
20
+ export interface ExecuteOutput {
21
+ status: 'success' | 'error';
22
+ screenshot?: ScreenshotOutput;
23
+ errorMessage?: string;
24
+ actions?: string[];
25
+ }
26
+ export declare const ACTION_SPACES: string[];
27
+ export declare class DesktopOperator {
28
+ static MANUAL: {
29
+ ACTION_SPACES: string[];
30
+ };
31
+ private config;
32
+ private mouseSpeed;
33
+ private screenContext;
34
+ constructor(config?: DesktopOperatorConfig);
35
+ takeScreenshot(): Promise<ScreenshotOutput>;
36
+ execute(params: ExecuteParams): Promise<ExecuteOutput>;
37
+ executeActionString(actionStr: string): Promise<void>;
38
+ private handleNavigate;
39
+ private handleFinished;
40
+ private parsePointInput;
41
+ private handleClickWithPoint;
42
+ private handleDoubleClickWithPoint;
43
+ private handleRightClickWithPoint;
44
+ private handleClick;
45
+ private handleDoubleClick;
46
+ private handleRightClick;
47
+ private handleDrag;
48
+ private handleType;
49
+ private handleHotkey;
50
+ private handleScroll;
51
+ private handleWait;
52
+ private parseBoxInput;
53
+ private parseKey;
54
+ }
55
+ //# sourceMappingURL=desktop-operator.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"desktop-operator.d.ts","sourceRoot":"","sources":["../../../src/gui-subagent/operator/desktop-operator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAiBH,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC;IAC5B,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AASD,eAAO,MAAM,aAAa,UAWzB,CAAC;AAmPF,qBAAa,eAAe;IAC1B,MAAM,CAAC,MAAM;;MAAqB;IAElC,OAAO,CAAC,MAAM,CAAkC;IAChD,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,aAAa,CAAgB;gBAEzB,MAAM,GAAE,qBAA0B;IAUxC,cAAc,IAAI,OAAO,CAAC,gBAAgB,CAAC;IAyB3C,OAAO,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAuBtD,mBAAmB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;YA4D7C,cAAc;YAed,cAAc;IAM5B,OAAO,CAAC,eAAe;YAKT,oBAAoB;YAWpB,0BAA0B;YAS1B,yBAAyB;YASzB,WAAW;YAQX,iBAAiB;YAQjB,gBAAgB;YAQhB,UAAU;YAcV,UAAU;YAqBV,YAAY;YA6BZ,YAAY;YAsBZ,UAAU;IAKxB,OAAO,CAAC,aAAa;IAerB,OAAO,CAAC,QAAQ;CA2BjB"}