@phuetz/code-buddy 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (312) hide show
  1. package/README.md +160 -164
  2. package/dist/agent/autonomous/agentic-coding-contract.d.ts +6 -6
  3. package/dist/agent/codebuddy-agent.d.ts +1 -1
  4. package/dist/agent/codebuddy-agent.js +101 -13
  5. package/dist/agent/execution/agent-executor.js +122 -12
  6. package/dist/agent/execution/tool-dependency-graph.js +7 -0
  7. package/dist/agent/execution/tool-hooks.d.ts +6 -0
  8. package/dist/agent/execution/tool-hooks.js +13 -5
  9. package/dist/agent/execution/tool-selection-strategy.d.ts +11 -2
  10. package/dist/agent/execution/tool-selection-strategy.js +67 -10
  11. package/dist/agent/extended-thinking.d.ts +8 -0
  12. package/dist/agent/extended-thinking.js +27 -0
  13. package/dist/agent/hermes-browser-backends.js +151 -11
  14. package/dist/agent/hermes-claw-migrate.d.ts +59 -0
  15. package/dist/agent/hermes-claw-migrate.js +427 -26
  16. package/dist/agent/hermes-parity-manifest.js +78 -71
  17. package/dist/agent/hermes-runtime-lifecycle.d.ts +67 -0
  18. package/dist/agent/hermes-runtime-lifecycle.js +468 -0
  19. package/dist/agent/lesson-auto-proposer.js +10 -0
  20. package/dist/agent/middleware/index.d.ts +1 -0
  21. package/dist/agent/middleware/index.js +1 -0
  22. package/dist/agent/middleware/session-duration.d.ts +36 -0
  23. package/dist/agent/middleware/session-duration.js +78 -0
  24. package/dist/agent/middleware/visual-validation-middleware.d.ts +17 -0
  25. package/dist/agent/middleware/visual-validation-middleware.js +67 -0
  26. package/dist/agent/model-benchmark.d.ts +77 -0
  27. package/dist/agent/model-benchmark.js +309 -0
  28. package/dist/agent/model-tier.d.ts +14 -0
  29. package/dist/agent/model-tier.js +71 -0
  30. package/dist/agent/multi-agent/agents/orchestrator-agent.js +17 -6
  31. package/dist/agent/multi-agent/multi-agent-system.js +3 -1
  32. package/dist/agent/multi-agent/session-fleet-bridge.d.ts +2 -0
  33. package/dist/agent/multi-agent/session-fleet-bridge.js +9 -1
  34. package/dist/agent/multi-agent/session-tools.js +22 -5
  35. package/dist/agent/reasoning/mcts.js +20 -6
  36. package/dist/agent/session-end-flush.d.ts +71 -0
  37. package/dist/agent/session-end-flush.js +233 -0
  38. package/dist/agent/streaming/streaming-handler.d.ts +7 -0
  39. package/dist/agent/streaming/streaming-handler.js +8 -1
  40. package/dist/agent/tool-executor.js +1 -0
  41. package/dist/agent/tool-handler.d.ts +5 -0
  42. package/dist/agent/tool-handler.js +130 -4
  43. package/dist/browser-automation/browser-operator-executor.js +15 -0
  44. package/dist/browser-automation/browser-use-runner.d.ts +96 -0
  45. package/dist/browser-automation/browser-use-runner.js +492 -0
  46. package/dist/browser-automation/camofox-runner.d.ts +107 -0
  47. package/dist/browser-automation/camofox-runner.js +287 -0
  48. package/dist/channels/dingtalk/index.js +3 -0
  49. package/dist/channels/discord/client.d.ts +9 -0
  50. package/dist/channels/discord/client.js +12 -0
  51. package/dist/channels/feishu/index.d.ts +167 -1
  52. package/dist/channels/feishu/index.js +432 -7
  53. package/dist/channels/gateway-lifecycle.d.ts +132 -0
  54. package/dist/channels/gateway-lifecycle.js +219 -0
  55. package/dist/channels/google-chat/index.d.ts +5 -1
  56. package/dist/channels/google-chat/index.js +5 -1
  57. package/dist/channels/imessage/index.d.ts +14 -0
  58. package/dist/channels/imessage/index.js +64 -17
  59. package/dist/channels/index.d.ts +4 -0
  60. package/dist/channels/index.js +4 -0
  61. package/dist/channels/irc/index.d.ts +73 -3
  62. package/dist/channels/irc/index.js +446 -11
  63. package/dist/channels/line/index.js +3 -0
  64. package/dist/channels/mattermost/index.d.ts +51 -4
  65. package/dist/channels/mattermost/index.js +303 -20
  66. package/dist/channels/nextcloud-talk/index.d.ts +89 -8
  67. package/dist/channels/nextcloud-talk/index.js +367 -16
  68. package/dist/channels/nostr/index.d.ts +121 -1
  69. package/dist/channels/nostr/index.js +396 -8
  70. package/dist/channels/ntfy/index.js +3 -0
  71. package/dist/channels/qq/index.js +3 -0
  72. package/dist/channels/slash-parity.d.ts +117 -0
  73. package/dist/channels/slash-parity.js +185 -0
  74. package/dist/channels/synology-chat/index.js +3 -0
  75. package/dist/channels/teams/index.d.ts +3 -0
  76. package/dist/channels/teams/index.js +3 -0
  77. package/dist/channels/telegram/client.d.ts +9 -0
  78. package/dist/channels/telegram/client.js +12 -0
  79. package/dist/channels/twilio-voice/index.js +3 -0
  80. package/dist/channels/webchat/index.d.ts +7 -1
  81. package/dist/channels/webchat/index.js +7 -1
  82. package/dist/channels/wecom/index.js +3 -0
  83. package/dist/channels/weixin/index.js +3 -0
  84. package/dist/channels/zalo/index.js +3 -0
  85. package/dist/cli/config-loader.js +1 -1
  86. package/dist/codebuddy/client.d.ts +30 -1
  87. package/dist/codebuddy/client.js +167 -20
  88. package/dist/codebuddy/providers/provider-chatgpt-responses.d.ts +5 -0
  89. package/dist/codebuddy/providers/provider-chatgpt-responses.js +1 -0
  90. package/dist/codebuddy/providers/provider-gemini-native.js +1 -1
  91. package/dist/codebuddy/providers/provider-openai-compat.d.ts +5 -0
  92. package/dist/codebuddy/providers/provider-openai-compat.js +82 -2
  93. package/dist/codebuddy/tool-definitions/computer-control-tools.js +32 -2
  94. package/dist/codebuddy/tool-definitions/core-tools.js +3 -3
  95. package/dist/codebuddy/tool-definitions/index.d.ts +2 -1
  96. package/dist/codebuddy/tool-definitions/index.js +2 -1
  97. package/dist/codebuddy/tool-definitions/multimodal-tools.d.ts +1 -0
  98. package/dist/codebuddy/tool-definitions/multimodal-tools.js +44 -1
  99. package/dist/codebuddy/tool-definitions/windows-tools.d.ts +2 -0
  100. package/dist/codebuddy/tool-definitions/windows-tools.js +34 -0
  101. package/dist/codebuddy/tools.d.ts +1 -1
  102. package/dist/codebuddy/tools.js +27 -24
  103. package/dist/commands/cli/hermes-commands.js +108 -0
  104. package/dist/commands/cli/native-engine-commands.js +107 -3
  105. package/dist/commands/cli/utility-commands.js +45 -0
  106. package/dist/commands/enhanced-command-handler.js +5 -0
  107. package/dist/commands/goal-cli.d.ts +71 -0
  108. package/dist/commands/goal-cli.js +280 -0
  109. package/dist/commands/handlers/goal-handler.d.ts +34 -0
  110. package/dist/commands/handlers/goal-handler.js +155 -0
  111. package/dist/commands/handlers/index.d.ts +1 -0
  112. package/dist/commands/handlers/index.js +2 -0
  113. package/dist/commands/handlers/infra-handlers.js +36 -0
  114. package/dist/commands/handlers/memory-handlers.js +96 -1
  115. package/dist/commands/headless-slash.d.ts +2 -0
  116. package/dist/commands/headless-slash.js +12 -0
  117. package/dist/commands/llm-provider-resolution.d.ts +5 -3
  118. package/dist/commands/llm-provider-resolution.js +87 -33
  119. package/dist/commands/ollama.d.ts +25 -0
  120. package/dist/commands/ollama.js +100 -0
  121. package/dist/commands/provider.d.ts +5 -0
  122. package/dist/commands/provider.js +106 -55
  123. package/dist/commands/slash/builtin-commands.js +20 -0
  124. package/dist/commands/spec-next.js +2 -1
  125. package/dist/commands/spec-plan.js +5 -16
  126. package/dist/commands/tunnel.d.ts +2 -0
  127. package/dist/commands/tunnel.js +48 -0
  128. package/dist/config/config-resolver.d.ts +2 -1
  129. package/dist/config/config-resolver.js +54 -41
  130. package/dist/config/constants.d.ts +28 -0
  131. package/dist/config/constants.js +7 -0
  132. package/dist/config/env-schema.js +770 -0
  133. package/dist/config/feature-flags.js +7 -0
  134. package/dist/config/model-tools.js +4 -4
  135. package/dist/config/toml-config.d.ts +16 -0
  136. package/dist/config/toml-config.js +3 -0
  137. package/dist/context/context-manager-v2.d.ts +39 -0
  138. package/dist/context/context-manager-v2.js +91 -0
  139. package/dist/daemon/agent-task-executor.js +12 -1
  140. package/dist/daemon/autonomous-daemon.d.ts +1 -1
  141. package/dist/daemon/autonomous-daemon.js +5 -3
  142. package/dist/daemon/autonomous-loop.d.ts +21 -1
  143. package/dist/daemon/autonomous-loop.js +69 -0
  144. package/dist/daemon/colab-goal.d.ts +38 -0
  145. package/dist/daemon/colab-goal.js +81 -0
  146. package/dist/daemon/cron-agent-bridge.d.ts +12 -3
  147. package/dist/daemon/cron-agent-bridge.js +25 -9
  148. package/dist/desktop/codebuddy-engine-adapter.d.ts +32 -0
  149. package/dist/desktop/codebuddy-engine-adapter.js +257 -80
  150. package/dist/desktop/engine-adapter.d.ts +14 -0
  151. package/dist/desktop-automation/automation-manager.js +16 -0
  152. package/dist/desktop-automation/omniparser-runner.d.ts +63 -0
  153. package/dist/desktop-automation/omniparser-runner.js +115 -0
  154. package/dist/desktop-automation/smart-snapshot.d.ts +11 -0
  155. package/dist/desktop-automation/smart-snapshot.js +72 -5
  156. package/dist/doctor/index.js +40 -23
  157. package/dist/events/types.d.ts +11 -0
  158. package/dist/fleet/colab-store.d.ts +21 -0
  159. package/dist/fleet/colab-store.js +28 -0
  160. package/dist/fleet/peer-session-bridge.d.ts +1 -1
  161. package/dist/fleet/peer-session-bridge.js +243 -2
  162. package/dist/fleet/peer-session-store.d.ts +3 -0
  163. package/dist/fleet/peer-tool-bridge.js +14 -0
  164. package/dist/fleet/privacy-lint.d.ts +8 -0
  165. package/dist/fleet/privacy-lint.js +22 -0
  166. package/dist/goals/goal-decomposer.d.ts +27 -0
  167. package/dist/goals/goal-decomposer.js +293 -0
  168. package/dist/goals/goal-judge-client.d.ts +15 -0
  169. package/dist/goals/goal-judge-client.js +45 -0
  170. package/dist/goals/goal-judge.d.ts +38 -0
  171. package/dist/goals/goal-judge.js +158 -0
  172. package/dist/goals/goal-loop.d.ts +37 -0
  173. package/dist/goals/goal-loop.js +101 -0
  174. package/dist/goals/goal-manager.d.ts +77 -0
  175. package/dist/goals/goal-manager.js +289 -0
  176. package/dist/goals/goal-state.d.ts +99 -0
  177. package/dist/goals/goal-state.js +315 -0
  178. package/dist/goals/goal-store.d.ts +28 -0
  179. package/dist/goals/goal-store.js +87 -0
  180. package/dist/goals/index.d.ts +5 -0
  181. package/dist/goals/index.js +6 -0
  182. package/dist/hooks/use-input-handler.js +36 -1
  183. package/dist/hooks/user-hooks.js +17 -3
  184. package/dist/index.js +506 -25
  185. package/dist/input/text-to-speech.d.ts +2 -6
  186. package/dist/input/text-to-speech.js +2 -27
  187. package/dist/integrations/screenpipe/screenpipe-client.d.ts +10 -0
  188. package/dist/integrations/screenpipe/screenpipe-client.js +12 -2
  189. package/dist/integrations/tailscale.d.ts +13 -0
  190. package/dist/integrations/tailscale.js +87 -34
  191. package/dist/knowledge/workspace-indexer.js +53 -9
  192. package/dist/mcp/client.js +1 -0
  193. package/dist/mcp/mcp-desktop-tools.d.ts +23 -0
  194. package/dist/mcp/mcp-desktop-tools.js +158 -0
  195. package/dist/mcp/mcp-server.js +2 -0
  196. package/dist/memory/enhanced-memory.d.ts +10 -0
  197. package/dist/memory/enhanced-memory.js +33 -2
  198. package/dist/memory/index.d.ts +3 -1
  199. package/dist/memory/index.js +4 -1
  200. package/dist/memory/memory-auto-proposer.d.ts +23 -0
  201. package/dist/memory/memory-auto-proposer.js +308 -0
  202. package/dist/memory/memory-candidate-queue.d.ts +92 -0
  203. package/dist/memory/memory-candidate-queue.js +261 -0
  204. package/dist/memory/persistent-memory.d.ts +45 -2
  205. package/dist/memory/persistent-memory.js +236 -40
  206. package/dist/ml/bayesian-qualifier.d.ts +1 -1
  207. package/dist/ml/bayesian-qualifier.js +35 -2
  208. package/dist/observability/run-store.d.ts +1 -1
  209. package/dist/openclaw/gateway-bridge.js +5 -0
  210. package/dist/plugins/marketplace.d.ts +1 -0
  211. package/dist/plugins/marketplace.js +7 -0
  212. package/dist/prompts/execution-discipline.d.ts +14 -0
  213. package/dist/prompts/execution-discipline.js +29 -0
  214. package/dist/prompts/variation-injector.js +9 -2
  215. package/dist/protocols/acp/acp-agentic-runner.d.ts +7 -6
  216. package/dist/protocols/acp/acp-agentic-runner.js +115 -18
  217. package/dist/protocols/acp/acp-session-store.d.ts +23 -0
  218. package/dist/protocols/acp/acp-session-store.js +77 -0
  219. package/dist/protocols/acp/acp-stdio-server.d.ts +7 -0
  220. package/dist/protocols/acp/acp-stdio-server.js +71 -16
  221. package/dist/providers/active-llm-registry.d.ts +37 -0
  222. package/dist/providers/active-llm-registry.js +186 -0
  223. package/dist/providers/auxiliary-provider.d.ts +25 -0
  224. package/dist/providers/auxiliary-provider.js +192 -0
  225. package/dist/providers/codex-oauth.d.ts +1 -1
  226. package/dist/providers/codex-oauth.js +27 -4
  227. package/dist/providers/index.d.ts +3 -0
  228. package/dist/providers/index.js +4 -0
  229. package/dist/providers/provider-catalog.d.ts +70 -0
  230. package/dist/providers/provider-catalog.js +738 -0
  231. package/dist/providers/provider-fallback.d.ts +35 -0
  232. package/dist/providers/provider-fallback.js +171 -0
  233. package/dist/providers/xai-oauth.d.ts +128 -0
  234. package/dist/providers/xai-oauth.js +735 -0
  235. package/dist/scheduler/cron-scheduler.d.ts +15 -3
  236. package/dist/scheduler/cron-scheduler.js +53 -7
  237. package/dist/scheduler/script-runner.d.ts +2 -0
  238. package/dist/scheduler/script-runner.js +19 -4
  239. package/dist/scheduler/watchdog-handlers.js +7 -20
  240. package/dist/search/usearch-index.js +7 -2
  241. package/dist/security/tool-policy/tool-groups.js +2 -0
  242. package/dist/server/index.js +17 -2
  243. package/dist/server/routes/mobile.d.ts +12 -4
  244. package/dist/server/routes/mobile.js +116 -1
  245. package/dist/server/tls-config.d.ts +35 -0
  246. package/dist/server/tls-config.js +142 -0
  247. package/dist/server/tunnel-manager.d.ts +20 -0
  248. package/dist/server/tunnel-manager.js +58 -0
  249. package/dist/server/websocket/fleet-bridge.d.ts +13 -1
  250. package/dist/server/websocket/fleet-bridge.js +16 -0
  251. package/dist/services/prompt-builder.d.ts +1 -0
  252. package/dist/services/prompt-builder.js +49 -8
  253. package/dist/shared/engine-types.d.ts +15 -1
  254. package/dist/sidecar/sidecar-bridge.d.ts +1 -0
  255. package/dist/sidecar/sidecar-bridge.js +21 -10
  256. package/dist/spec/spec-store.js +8 -1
  257. package/dist/talk-mode/providers/audioreader-tts.d.ts +1 -1
  258. package/dist/talk-mode/providers/audioreader-tts.js +8 -8
  259. package/dist/templates/project-scaffolding.js +3 -2
  260. package/dist/themes/theme-schema.d.ts +10 -10
  261. package/dist/tools/application-profiles.js +38 -0
  262. package/dist/tools/bash/bash-tool.d.ts +1 -0
  263. package/dist/tools/bash/bash-tool.js +16 -2
  264. package/dist/tools/bash/command-validator.js +3 -0
  265. package/dist/tools/bash/rtk-rewrite.d.ts +16 -0
  266. package/dist/tools/bash/rtk-rewrite.js +100 -0
  267. package/dist/tools/bash/streaming-executor.js +14 -1
  268. package/dist/tools/computer-control-tool.d.ts +16 -1
  269. package/dist/tools/computer-control-tool.js +317 -5
  270. package/dist/tools/document-generator.d.ts +14 -0
  271. package/dist/tools/document-generator.js +79 -0
  272. package/dist/tools/execute-code-rpc-invoker.d.ts +26 -2
  273. package/dist/tools/execute-code-rpc-invoker.js +46 -5
  274. package/dist/tools/metadata.js +28 -0
  275. package/dist/tools/office-macro-tool.d.ts +10 -0
  276. package/dist/tools/office-macro-tool.js +93 -0
  277. package/dist/tools/registry/index.d.ts +2 -1
  278. package/dist/tools/registry/index.js +2 -1
  279. package/dist/tools/registry/memory-tools.d.ts +20 -0
  280. package/dist/tools/registry/memory-tools.js +225 -3
  281. package/dist/tools/registry/multimodal-tools.js +2 -2
  282. package/dist/tools/registry/vision-tools.d.ts +33 -0
  283. package/dist/tools/registry/vision-tools.js +308 -6
  284. package/dist/tools/registry/windows-tools.d.ts +2 -0
  285. package/dist/tools/registry/windows-tools.js +47 -0
  286. package/dist/tools/route-peer-tool.js +14 -0
  287. package/dist/tools/screenshot-tool.js +14 -2
  288. package/dist/tools/text-to-speech-tool.d.ts +1 -1
  289. package/dist/tools/text-to-speech-tool.js +2 -12
  290. package/dist/tools/tool-selector.js +3 -59
  291. package/dist/utils/config-validation/schema.d.ts +40 -7
  292. package/dist/utils/config-validation/schema.js +130 -1
  293. package/dist/utils/cost-tracker.js +21 -1
  294. package/dist/utils/disk-guard.d.ts +180 -0
  295. package/dist/utils/disk-guard.js +404 -0
  296. package/dist/utils/interactive-setup.js +35 -2
  297. package/dist/utils/model-utils.js +11 -2
  298. package/dist/utils/output-sanitizer.js +11 -0
  299. package/dist/utils/provider-detector.d.ts +8 -7
  300. package/dist/utils/provider-detector.js +19 -77
  301. package/dist/utils/settings-manager.d.ts +7 -0
  302. package/dist/utils/token-display.d.ts +3 -3
  303. package/dist/utils/token-display.js +22 -4
  304. package/dist/wizard/onboarding.d.ts +39 -0
  305. package/dist/wizard/onboarding.js +433 -21
  306. package/dist/wizard/provider-onboarding.d.ts +8 -23
  307. package/dist/wizard/provider-onboarding.js +39 -158
  308. package/package.json +4 -2
  309. package/dist/plugins/provider-onboarding.d.ts +0 -23
  310. package/dist/plugins/provider-onboarding.js +0 -116
  311. package/dist/utils/kokoro-tts.d.ts +0 -8
  312. package/dist/utils/kokoro-tts.js +0 -67
@@ -0,0 +1,63 @@
1
+ export interface OmniParserElement {
2
+ /** Index in the server's parsed_content_list — this is the number drawn on the annotated (SOM) image. */
3
+ id: number;
4
+ /** 'text' | 'icon' (whatever the server reports). */
5
+ type: string;
6
+ /** OCR text or icon caption (the server's `content`). */
7
+ content: string;
8
+ /** Whether OmniParser flagged the element as interactive/clickable (`interactivity`). */
9
+ interactable: boolean;
10
+ /** Bounding box [x1, y1, x2, y2]. Pixels when `normalized` is false, else 0-1 ratios. */
11
+ bbox: [number, number, number, number];
12
+ /** Box center [x, y] — pixels when `normalized` is false. Handy for a follow-up click. */
13
+ center: [number, number];
14
+ /** True when bbox/center are 0-1 ratios (image dimensions were unknown). */
15
+ normalized: boolean;
16
+ }
17
+ export interface OmniParserResult {
18
+ elements: OmniParserElement[];
19
+ /** Base64 of the Set-of-Marks (numbered) image returned by the server, or the original on failure. */
20
+ annotatedImageBase64: string;
21
+ }
22
+ export interface ParseScreenOptions {
23
+ /** Screenshot pixel width — when provided, normalized boxes are scaled to pixels. */
24
+ width?: number;
25
+ /** Screenshot pixel height. */
26
+ height?: number;
27
+ }
28
+ /**
29
+ * Runner for OmniParser (Microsoft's screen-parsing model — github.com/microsoft/OmniParser).
30
+ *
31
+ * Talks to the OmniParser v2 `omniparserserver` FastAPI service (omnitool/omniparserserver):
32
+ * - POST {base}/parse/ body {"base64_image": "<b64>"}
33
+ * -> {"som_image_base64": "<b64>", "parsed_content_list": [{type, content, interactivity, bbox}], "latency"}
34
+ * - GET {base}/probe/ -> health check
35
+ *
36
+ * `bbox` from the server is normalized to 0-1; we scale it to pixels when the
37
+ * screenshot dimensions are known so the coordinates are directly clickable.
38
+ *
39
+ * The model is GPU/Python and must be self-hosted; configure the base URL via
40
+ * OMNIPARSER_API_URL (default http://localhost:8000). Off-server, every call
41
+ * degrades gracefully to empty elements + the original screenshot.
42
+ */
43
+ export declare class OmniParserRunner {
44
+ private baseUrl;
45
+ private apiKey;
46
+ constructor();
47
+ private get parseUrl();
48
+ private get probeUrl();
49
+ /**
50
+ * Send a base64-encoded screenshot to the OmniParser server for analysis.
51
+ *
52
+ * @param imageBase64 The raw screenshot in base64 (no data: prefix).
53
+ * @param opts Optional screenshot pixel dimensions used to scale normalized boxes.
54
+ * @returns Parsed elements and the annotated (Set-of-Marks) image.
55
+ */
56
+ parseScreen(imageBase64: string, opts?: ParseScreenOptions): Promise<OmniParserResult>;
57
+ /** Map OmniParser's parsed_content_list into our element shape, scaling boxes to pixels when possible. */
58
+ private mapElements;
59
+ /**
60
+ * Check if the OmniParser server is reachable via its /probe/ health endpoint.
61
+ */
62
+ isAvailable(): Promise<boolean>;
63
+ }
@@ -0,0 +1,115 @@
1
+ import { logger } from '../utils/logger.js';
2
+ /**
3
+ * Runner for OmniParser (Microsoft's screen-parsing model — github.com/microsoft/OmniParser).
4
+ *
5
+ * Talks to the OmniParser v2 `omniparserserver` FastAPI service (omnitool/omniparserserver):
6
+ * - POST {base}/parse/ body {"base64_image": "<b64>"}
7
+ * -> {"som_image_base64": "<b64>", "parsed_content_list": [{type, content, interactivity, bbox}], "latency"}
8
+ * - GET {base}/probe/ -> health check
9
+ *
10
+ * `bbox` from the server is normalized to 0-1; we scale it to pixels when the
11
+ * screenshot dimensions are known so the coordinates are directly clickable.
12
+ *
13
+ * The model is GPU/Python and must be self-hosted; configure the base URL via
14
+ * OMNIPARSER_API_URL (default http://localhost:8000). Off-server, every call
15
+ * degrades gracefully to empty elements + the original screenshot.
16
+ */
17
+ export class OmniParserRunner {
18
+ baseUrl;
19
+ apiKey;
20
+ constructor() {
21
+ // OMNIPARSER_API_URL is the server base (scheme://host:port); endpoints are derived.
22
+ const raw = process.env.OMNIPARSER_API_URL || 'http://localhost:8000';
23
+ this.baseUrl = raw.replace(/\/+$/, '');
24
+ this.apiKey = process.env.OMNIPARSER_API_KEY;
25
+ }
26
+ get parseUrl() {
27
+ return `${this.baseUrl}/parse/`;
28
+ }
29
+ get probeUrl() {
30
+ return `${this.baseUrl}/probe/`;
31
+ }
32
+ /**
33
+ * Send a base64-encoded screenshot to the OmniParser server for analysis.
34
+ *
35
+ * @param imageBase64 The raw screenshot in base64 (no data: prefix).
36
+ * @param opts Optional screenshot pixel dimensions used to scale normalized boxes.
37
+ * @returns Parsed elements and the annotated (Set-of-Marks) image.
38
+ */
39
+ async parseScreen(imageBase64, opts = {}) {
40
+ try {
41
+ logger.debug('Sending screenshot to OmniParser API', { url: this.parseUrl });
42
+ const headers = {
43
+ 'Content-Type': 'application/json',
44
+ };
45
+ if (this.apiKey) {
46
+ headers['Authorization'] = `Bearer ${this.apiKey}`;
47
+ }
48
+ const response = await fetch(this.parseUrl, {
49
+ method: 'POST',
50
+ headers,
51
+ // OmniParser v2 ParseRequest expects the field `base64_image`.
52
+ body: JSON.stringify({ base64_image: imageBase64 }),
53
+ });
54
+ if (!response.ok) {
55
+ throw new Error(`OmniParser API returned status: ${response.status} ${response.statusText}`);
56
+ }
57
+ const data = (await response.json());
58
+ const elements = this.mapElements(data.parsed_content_list ?? [], opts);
59
+ return {
60
+ elements,
61
+ annotatedImageBase64: data.som_image_base64 || imageBase64,
62
+ };
63
+ }
64
+ catch (error) {
65
+ const msg = error instanceof Error ? error.message : String(error);
66
+ logger.error('OmniParser API call failed', { error: msg, url: this.parseUrl });
67
+ // Graceful fallback if the server is down/misconfigured: original image, no elements.
68
+ return {
69
+ elements: [],
70
+ annotatedImageBase64: imageBase64,
71
+ };
72
+ }
73
+ }
74
+ /** Map OmniParser's parsed_content_list into our element shape, scaling boxes to pixels when possible. */
75
+ mapElements(raw, opts) {
76
+ const { width, height } = opts;
77
+ const scalable = typeof width === 'number' && width > 0 && typeof height === 'number' && height > 0;
78
+ return raw.map((item, index) => {
79
+ const b = Array.isArray(item.bbox) && item.bbox.length === 4 ? item.bbox : [0, 0, 0, 0];
80
+ let [x1, y1, x2, y2] = [Number(b[0]) || 0, Number(b[1]) || 0, Number(b[2]) || 0, Number(b[3]) || 0];
81
+ if (scalable) {
82
+ x1 = Math.round(x1 * width);
83
+ y1 = Math.round(y1 * height);
84
+ x2 = Math.round(x2 * width);
85
+ y2 = Math.round(y2 * height);
86
+ }
87
+ return {
88
+ id: index,
89
+ type: item.type ?? 'unknown',
90
+ content: item.content ?? '',
91
+ interactable: item.interactivity ?? false,
92
+ bbox: [x1, y1, x2, y2],
93
+ center: [Math.round((x1 + x2) / 2), Math.round((y1 + y2) / 2)],
94
+ normalized: !scalable,
95
+ };
96
+ });
97
+ }
98
+ /**
99
+ * Check if the OmniParser server is reachable via its /probe/ health endpoint.
100
+ */
101
+ async isAvailable() {
102
+ try {
103
+ const headers = {};
104
+ if (this.apiKey) {
105
+ headers['Authorization'] = `Bearer ${this.apiKey}`;
106
+ }
107
+ const response = await fetch(this.probeUrl, { method: 'GET', headers });
108
+ return response.ok;
109
+ }
110
+ catch {
111
+ return false;
112
+ }
113
+ }
114
+ }
115
+ //# sourceMappingURL=omniparser-runner.js.map
@@ -132,6 +132,8 @@ export declare class SmartSnapshotManager extends EventEmitter {
132
132
  private snapshotCache;
133
133
  private static readonly MAX_CACHE_SIZE;
134
134
  private nextRef;
135
+ /** Cached python interpreter that has the Atspi GI binding (`''` = none found, `null` = not yet probed). */
136
+ private atspiPython;
135
137
  constructor(config?: Partial<SmartSnapshotConfig>);
136
138
  private runPowerShellEncoded;
137
139
  /**
@@ -206,6 +208,15 @@ export declare class SmartSnapshotManager extends EventEmitter {
206
208
  * Detect elements on WSL2 via PowerShell UIAutomation (same script as detectWindowsElements)
207
209
  */
208
210
  private detectWSLElements;
211
+ /**
212
+ * Resolve a python3 interpreter that actually has the Atspi GObject-Introspection
213
+ * binding. `python3` on PATH is frequently a conda/pyenv interpreter that lacks the
214
+ * system `gi`/Atspi typelib (PyGObject is installed against the system python), so a
215
+ * bare `python3 -c "import gi"` silently fails and the snapshot falls back to mock
216
+ * elements. We probe a small candidate list once and cache the winner.
217
+ * Returns '' when no interpreter has the binding.
218
+ */
219
+ private resolveAtspiPython;
209
220
  /**
210
221
  * Detect elements on native Linux via AT-SPI
211
222
  */
@@ -43,6 +43,8 @@ export class SmartSnapshotManager extends EventEmitter {
43
43
  snapshotCache = new Map();
44
44
  static MAX_CACHE_SIZE = 20;
45
45
  nextRef = 1;
46
+ /** Cached python interpreter that has the Atspi GI binding (`''` = none found, `null` = not yet probed). */
47
+ atspiPython = null;
46
48
  constructor(config = {}) {
47
49
  super();
48
50
  this.config = { ...DEFAULT_CONFIG, ...config };
@@ -678,17 +680,76 @@ if ($focused) {
678
680
  }
679
681
  return elements;
680
682
  }
683
+ /**
684
+ * Resolve a python3 interpreter that actually has the Atspi GObject-Introspection
685
+ * binding. `python3` on PATH is frequently a conda/pyenv interpreter that lacks the
686
+ * system `gi`/Atspi typelib (PyGObject is installed against the system python), so a
687
+ * bare `python3 -c "import gi"` silently fails and the snapshot falls back to mock
688
+ * elements. We probe a small candidate list once and cache the winner.
689
+ * Returns '' when no interpreter has the binding.
690
+ */
691
+ resolveAtspiPython() {
692
+ if (this.atspiPython !== null) {
693
+ return this.atspiPython;
694
+ }
695
+ const candidates = [
696
+ process.env.CODEBUDDY_ATSPI_PYTHON,
697
+ 'python3',
698
+ '/usr/bin/python3',
699
+ 'python',
700
+ ].filter((c) => typeof c === 'string' && c.length > 0);
701
+ // Probe the *exact* requirement — a python can have `gi` yet lack the Atspi typelib.
702
+ const probe = `import gi; gi.require_version('Atspi','2.0'); from gi.repository import Atspi; print('OK')`;
703
+ let transientFailure = false;
704
+ for (const candidate of candidates) {
705
+ try {
706
+ const out = execSync(`${candidate} -c "${probe}" 2>/dev/null`, {
707
+ encoding: 'utf-8',
708
+ timeout: 5000,
709
+ }).trim();
710
+ if (out.includes('OK')) {
711
+ this.atspiPython = candidate;
712
+ return candidate;
713
+ }
714
+ }
715
+ catch (err) {
716
+ // Distinguish a transient timeout/kill (machine under load) from a genuine
717
+ // missing interpreter/binding. A timeout must NOT poison the cache, or a
718
+ // single slow probe would silently downgrade every later snapshot to mock
719
+ // until the process restarts.
720
+ const e = err;
721
+ if (e?.signal === 'SIGTERM' || e?.code === 'ETIMEDOUT')
722
+ transientFailure = true;
723
+ }
724
+ }
725
+ if (transientFailure) {
726
+ // Leave the cache unset (null) so the next snapshot re-probes once load clears.
727
+ return '';
728
+ }
729
+ // Every candidate cleanly failed (absent/binding missing) — cache to avoid
730
+ // re-probing 3 interpreters on every snapshot.
731
+ this.atspiPython = '';
732
+ return '';
733
+ }
681
734
  /**
682
735
  * Detect elements on native Linux via AT-SPI
683
736
  */
684
737
  async detectLinuxATSPIElements(_options) {
685
738
  const elements = [];
739
+ const python = this.resolveAtspiPython();
740
+ if (!python) {
741
+ logger.warn('AT-SPI element enumeration unavailable: no python3 interpreter with the gi/Atspi binding found. ' +
742
+ 'Install python3-gi + gir1.2-atspi-2.0, or set CODEBUDDY_ATSPI_PYTHON to a python that has them. ' +
743
+ 'Falling back to mock UI elements (NOT a real snapshot).');
744
+ elements.push(...this.getMockElements());
745
+ return elements;
746
+ }
686
747
  try {
687
748
  // Use AT-SPI via python-atspi or accerciser
688
749
  // This requires libatspi and python3-atspi2 to be installed
689
750
  const script = `
690
751
  import gi
691
- gi.require_version('Atspi', '2.0')
752
+ gi.require_version("Atspi", "2.0")
692
753
  from gi.repository import Atspi
693
754
  import json
694
755
 
@@ -733,9 +794,14 @@ for i in range(desktop.get_child_count()):
733
794
 
734
795
  print(json.dumps(all_elements[:100]))
735
796
  `;
736
- // Try to run the Python script
797
+ // Try to run the Python script. Pass it base64-encoded and decode inside the
798
+ // interpreter — embedding a multi-line script (which itself contains quotes)
799
+ // directly in a `-c '...'` shell argument breaks quoting and silently yields a
800
+ // NameError, masking real AT-SPI output behind the mock fallback.
737
801
  try {
738
- const { stdout } = await execAsync(`python3 -c '${script}' 2>/dev/null`, {
802
+ const encoded = Buffer.from(script, 'utf-8').toString('base64');
803
+ const runner = `import base64; exec(base64.b64decode("${encoded}").decode("utf-8"))`;
804
+ const { stdout } = await execAsync(`${python} -c '${runner}' 2>/dev/null`, {
739
805
  timeout: 10000,
740
806
  });
741
807
  const parsed = JSON.parse(stdout);
@@ -758,12 +824,13 @@ print(json.dumps(all_elements[:100]))
758
824
  }
759
825
  }
760
826
  catch (_err) {
761
- // Intentionally ignored: AT-SPI parsing may fail, fallback to mock elements
827
+ // AT-SPI query/parse failed even though a gi-capable python was found.
828
+ logger.warn('AT-SPI enumeration failed at query time; falling back to mock UI elements (NOT a real snapshot)', { error: _err });
762
829
  elements.push(...this.getMockElements());
763
830
  }
764
831
  }
765
832
  catch (error) {
766
- logger.debug('Linux accessibility detection failed', { error });
833
+ logger.warn('Linux accessibility detection failed; falling back to mock UI elements (NOT a real snapshot)', { error });
767
834
  elements.push(...this.getMockElements());
768
835
  }
769
836
  return elements;
@@ -1,7 +1,8 @@
1
1
  import { execSync } from 'child_process';
2
- import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, statfsSync, unlinkSync, writeFileSync, } from 'fs';
2
+ import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, unlinkSync, writeFileSync, } from 'fs';
3
3
  import { join } from 'path';
4
4
  import { logger } from '../utils/logger.js';
5
+ import { getFreeSpaceInfo } from '../utils/disk-guard.js';
5
6
  function commandExists(cmd) {
6
7
  try {
7
8
  const lookupCommand = process.platform === 'win32' ? `where ${cmd}` : `which ${cmd}`;
@@ -17,10 +18,34 @@ function getCommandAvailability(cmd) {
17
18
  }
18
19
  function checkNodeVersion() {
19
20
  const major = parseInt(process.version.slice(1), 10);
20
- if (major >= 18) {
21
- return { name: 'Node.js version', status: 'ok', message: `${process.version} (>= 18 required)` };
21
+ if (major < 18) {
22
+ return { name: 'Node.js version', status: 'error', message: `${process.version} — Node.js >= 18 is required` };
23
+ }
24
+ if (major < 22) {
25
+ return {
26
+ name: 'Node.js version',
27
+ status: 'warn',
28
+ message: `${process.version} — OK for the CLI (>= 18), but the Cowork desktop app needs >= 22`,
29
+ };
30
+ }
31
+ return { name: 'Node.js version', status: 'ok', message: `${process.version} (CLI >= 18 and Cowork >= 22 OK)` };
32
+ }
33
+ // The SQLite layer (memory/sessions/cache/analytics) is a native module. On a
34
+ // platform without a matching prebuilt binary, `npm install` may have failed to
35
+ // build it — surface that here clearly instead of crashing a DB feature later.
36
+ async function checkNativeSqlite() {
37
+ try {
38
+ await import('better-sqlite3');
39
+ return { name: 'SQLite (better-sqlite3)', status: 'ok', message: 'native module available' };
40
+ }
41
+ catch {
42
+ return {
43
+ name: 'SQLite (better-sqlite3)',
44
+ status: 'warn',
45
+ message: 'native module not built — DB-backed features (memory/sessions/cache) are unavailable. ' +
46
+ 'Install build tools and reinstall (Linux: `build-essential python3`; macOS: `xcode-select --install`).',
47
+ };
22
48
  }
23
- return { name: 'Node.js version', status: 'error', message: `${process.version} — Node.js >= 18 is required` };
24
49
  }
25
50
  function checkDependencies() {
26
51
  const checks = [];
@@ -170,33 +195,24 @@ function checkTtsProviders() {
170
195
  { cmd: 'espeak', label: 'espeak' },
171
196
  ];
172
197
  const found = providers.filter(p => commandExists(p.cmd));
173
- try {
174
- // Check if kokoro-js is installed in workspace node_modules
175
- import.meta.resolve?.('kokoro-js');
176
- found.push({ cmd: 'kokoro', label: 'kokoro (npm)' });
177
- }
178
- catch {
179
- // ignore
180
- }
181
198
  return [{
182
199
  name: 'TTS providers',
183
200
  status: found.length > 0 ? 'ok' : 'warn',
184
- message: found.length > 0 ? `available: ${found.map(p => p.label).join(', ')}` : 'none found (install edge-tts, espeak or use kokoro via npm)',
201
+ message: found.length > 0 ? `available: ${found.map(p => p.label).join(', ')}` : 'none found (install edge-tts, espeak)',
185
202
  }];
186
203
  }
187
204
  function checkDiskSpace(cwd) {
188
- try {
189
- const stats = statfsSync(cwd);
190
- const freeBytes = stats.bfree * stats.bsize;
191
- const freeGB = freeBytes / (1024 ** 3);
192
- if (freeGB < 1) {
193
- return { name: 'Disk space', status: 'warn', message: `${freeGB.toFixed(2)} GB free (< 1 GB)` };
194
- }
195
- return { name: 'Disk space', status: 'ok', message: `${freeGB.toFixed(1)} GB free` };
196
- }
197
- catch {
205
+ // Single source of truth (disk-guard). Uses bavail — free space actually
206
+ // available to a non-root process — rather than bfree.
207
+ const info = getFreeSpaceInfo(cwd);
208
+ if (info === null) {
198
209
  return { name: 'Disk space', status: 'warn', message: 'unable to check' };
199
210
  }
211
+ const freeGB = info.freeBytes / (1024 ** 3);
212
+ if (freeGB < 1) {
213
+ return { name: 'Disk space', status: 'warn', message: `${freeGB.toFixed(2)} GB free (< 1 GB)` };
214
+ }
215
+ return { name: 'Disk space', status: 'ok', message: `${freeGB.toFixed(1)} GB free` };
200
216
  }
201
217
  function checkGit(cwd) {
202
218
  if (!commandExists('git')) {
@@ -385,6 +401,7 @@ export async function runDoctorChecks(cwd) {
385
401
  const dir = cwd ?? process.cwd();
386
402
  return [
387
403
  checkNodeVersion(),
404
+ await checkNativeSqlite(),
388
405
  ...checkDependencies(),
389
406
  ...checkApiKeys(),
390
407
  await checkChatGptOAuth(),
@@ -629,6 +629,16 @@ export interface CostEvents extends Record<string, BaseEvent> {
629
629
  'cost:limit_reached': CostLimitReachedEvent;
630
630
  'cost:warning': CostWarningEvent;
631
631
  }
632
+ export interface FleetActivityEvent extends BaseEvent {
633
+ type: 'fleet:activity';
634
+ activityType: string;
635
+ title: string;
636
+ description: string;
637
+ metadata?: Record<string, unknown>;
638
+ }
639
+ export interface FleetEvents extends Record<string, BaseEvent> {
640
+ 'fleet:activity': FleetActivityEvent;
641
+ }
632
642
  /**
633
643
  * Complete map of all application events.
634
644
  * This is the master type for the unified event system.
@@ -723,6 +733,7 @@ export interface AllEvents extends Record<string, BaseEvent> {
723
733
  'sandbox:created': SandboxCreatedEvent;
724
734
  'sandbox:destroyed': SandboxDestroyedEvent;
725
735
  'sandbox:execution': SandboxExecutionEvent;
736
+ 'fleet:activity': FleetActivityEvent;
726
737
  'cost:updated': CostUpdatedEvent;
727
738
  'cost:limit_reached': CostLimitReachedEvent;
728
739
  'cost:warning': CostWarningEvent;
@@ -48,6 +48,20 @@ export interface ColabTask {
48
48
  verifyCommand?: string;
49
49
  /** Ids of tasks that must be `completed` before this one is claimable (DAG). */
50
50
  dependsOn?: string[];
51
+ /**
52
+ * Goal-mode (Hermes kanban goal-mode parity): the worker loops on this task
53
+ * — after each attempt an LLM judge checks the title/description (+
54
+ * `acceptanceCriteria` as strict criteria); "continue" re-opens the task
55
+ * with a continuation nudge until `goalMaxTurns` is spent, then it is
56
+ * BLOCKED for human review instead of spinning.
57
+ */
58
+ goalMode?: boolean;
59
+ /** Per-task turn budget for goal-mode (default 5). */
60
+ goalMaxTurns?: number;
61
+ /** Goal-mode turns consumed so far (persisted — survives daemon restarts). */
62
+ goalTurnsUsed?: number;
63
+ /** Last judge reason (shown in the next continuation nudge). */
64
+ goalLastReason?: string;
51
65
  createdBy?: string;
52
66
  createdAt?: string;
53
67
  }
@@ -104,6 +118,8 @@ export interface AddTaskInput {
104
118
  acceptanceCriteria?: string[];
105
119
  verifyCommand?: string;
106
120
  dependsOn?: string[];
121
+ goalMode?: boolean;
122
+ goalMaxTurns?: number;
107
123
  createdBy?: string;
108
124
  id?: string;
109
125
  }
@@ -164,6 +180,11 @@ export declare class FleetColabStore {
164
180
  };
165
181
  /** Block a task with a reason (needs human/unblocking). */
166
182
  blockTask(taskId: string, reason: string): ColabTask;
183
+ /**
184
+ * Record a consumed goal-mode turn (persisted, so the budget survives daemon
185
+ * restarts) and remember the judge's reason for the next continuation nudge.
186
+ */
187
+ recordGoalTurn(taskId: string, reason: string): ColabTask;
167
188
  /** Release a claimed task back to the open pool. */
168
189
  releaseTask(taskId: string): ColabTask;
169
190
  addTask(input: AddTaskInput): ColabTask;
@@ -41,6 +41,17 @@ export function defaultFleetAgentId(cwd = process.cwd()) {
41
41
  const repo = path.basename(path.resolve(cwd)) || 'repo';
42
42
  return `${host}/${repo}`;
43
43
  }
44
+ function resolveGoalMaxTurns(raw) {
45
+ if (raw === undefined)
46
+ return undefined;
47
+ if (typeof raw !== 'number' || !Number.isSafeInteger(raw) || raw <= 0) {
48
+ throw new Error('goalMaxTurns must be a positive integer');
49
+ }
50
+ return raw;
51
+ }
52
+ function normalizeGoalTurnsUsed(raw) {
53
+ return typeof raw === 'number' && Number.isSafeInteger(raw) && raw >= 0 ? raw : 0;
54
+ }
44
55
  export class FleetColabStore {
45
56
  dir;
46
57
  agentId;
@@ -207,6 +218,20 @@ export class FleetColabStore {
207
218
  this.writeTasks(file);
208
219
  return { ...task };
209
220
  }
221
+ /**
222
+ * Record a consumed goal-mode turn (persisted, so the budget survives daemon
223
+ * restarts) and remember the judge's reason for the next continuation nudge.
224
+ */
225
+ recordGoalTurn(taskId, reason) {
226
+ const file = this.readTasks();
227
+ const task = file.tasks.find((t) => t.id === taskId);
228
+ if (!task)
229
+ throw new Error(`Unknown fleet task '${taskId}'`);
230
+ task.goalTurnsUsed = normalizeGoalTurnsUsed(task.goalTurnsUsed) + 1;
231
+ task.goalLastReason = reason;
232
+ this.writeTasks(file);
233
+ return { ...task };
234
+ }
210
235
  /** Release a claimed task back to the open pool. */
211
236
  releaseTask(taskId) {
212
237
  const file = this.readTasks();
@@ -221,6 +246,7 @@ export class FleetColabStore {
221
246
  }
222
247
  addTask(input) {
223
248
  const file = this.readTasks();
249
+ const goalMaxTurns = resolveGoalMaxTurns(input.goalMaxTurns);
224
250
  const task = {
225
251
  id: input.id ?? this.generateId('task'),
226
252
  title: input.title,
@@ -234,6 +260,8 @@ export class FleetColabStore {
234
260
  ...(input.acceptanceCriteria ? { acceptanceCriteria: input.acceptanceCriteria } : {}),
235
261
  ...(input.verifyCommand ? { verifyCommand: input.verifyCommand } : {}),
236
262
  ...(input.dependsOn && input.dependsOn.length > 0 ? { dependsOn: [...new Set(input.dependsOn)] } : {}),
263
+ ...(input.goalMode ? { goalMode: true } : {}),
264
+ ...(goalMaxTurns !== undefined ? { goalMaxTurns } : {}),
237
265
  createdBy: input.createdBy ?? this.agentId,
238
266
  createdAt: this.isoNow(),
239
267
  };
@@ -42,7 +42,7 @@ export type PeerChatClientGetter = () => CodeBuddyClient | null;
42
42
  * getter; un-wire first if you need to swap).
43
43
  */
44
44
  export declare function wirePeerSessionBridge(getClient: PeerChatClientGetter): Promise<void>;
45
- /** Detach all three methods. Idempotent. Does NOT clear in-memory sessions. */
45
+ /** Detach all registered methods. Idempotent. Does NOT clear in-memory sessions. */
46
46
  export declare function unwirePeerSessionBridge(): void;
47
47
  /** Whether the bridge is currently registered on the peer-rpc registry. */
48
48
  export declare function isPeerSessionBridgeWired(): boolean;