crewly 1.20.40 → 1.20.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/config/skills/_common/desktop-guards.sh +485 -0
  2. package/config/skills/_common/desktop-guards.test.sh +242 -0
  3. package/config/skills/_common/desktop-perceive.swift +530 -0
  4. package/config/skills/_common/desktop-presence.swift +343 -0
  5. package/config/skills/agent/_common/desktop-guards.sh +4 -0
  6. package/config/skills/agent/computer-use/SKILL.md +88 -0
  7. package/config/skills/agent/computer-use/execute.sh +249 -3
  8. package/config/skills/agent/desktop-app-control/SKILL.md +19 -0
  9. package/config/skills/agent/remote-browser/SKILL.md +19 -0
  10. package/dist/backend/backend/src/controllers/desktop/desktop.controller.d.ts +105 -0
  11. package/dist/backend/backend/src/controllers/desktop/desktop.controller.d.ts.map +1 -0
  12. package/dist/backend/backend/src/controllers/desktop/desktop.controller.js +278 -0
  13. package/dist/backend/backend/src/controllers/desktop/desktop.controller.js.map +1 -0
  14. package/dist/backend/backend/src/controllers/desktop/desktop.routes.d.ts +21 -0
  15. package/dist/backend/backend/src/controllers/desktop/desktop.routes.d.ts.map +1 -0
  16. package/dist/backend/backend/src/controllers/desktop/desktop.routes.js +31 -0
  17. package/dist/backend/backend/src/controllers/desktop/desktop.routes.js.map +1 -0
  18. package/dist/backend/backend/src/routes/api.routes.d.ts.map +1 -1
  19. package/dist/backend/backend/src/routes/api.routes.js +3 -0
  20. package/dist/backend/backend/src/routes/api.routes.js.map +1 -1
  21. package/dist/backend/backend/src/services/cloud/mobile-api-relay.service.d.ts.map +1 -1
  22. package/dist/backend/backend/src/services/cloud/mobile-api-relay.service.js +9 -0
  23. package/dist/backend/backend/src/services/cloud/mobile-api-relay.service.js.map +1 -1
  24. package/dist/backend/backend/src/services/slack/slack-team-channel.service.d.ts +24 -0
  25. package/dist/backend/backend/src/services/slack/slack-team-channel.service.d.ts.map +1 -1
  26. package/dist/backend/backend/src/services/slack/slack-team-channel.service.js +60 -1
  27. package/dist/backend/backend/src/services/slack/slack-team-channel.service.js.map +1 -1
  28. package/dist/backend/backend/src/services/slack/slack.service.d.ts +17 -0
  29. package/dist/backend/backend/src/services/slack/slack.service.d.ts.map +1 -1
  30. package/dist/backend/backend/src/services/slack/slack.service.js +32 -0
  31. package/dist/backend/backend/src/services/slack/slack.service.js.map +1 -1
  32. package/dist/backend/backend/src/types/slack.types.d.ts +10 -0
  33. package/dist/backend/backend/src/types/slack.types.d.ts.map +1 -1
  34. package/dist/backend/backend/src/types/slack.types.js.map +1 -1
  35. package/dist/backend/backend/src/utils/incomplete-turn.utils.d.ts +1 -1
  36. package/dist/backend/backend/src/utils/incomplete-turn.utils.d.ts.map +1 -1
  37. package/dist/backend/backend/src/utils/incomplete-turn.utils.js +4 -0
  38. package/dist/backend/backend/src/utils/incomplete-turn.utils.js.map +1 -1
  39. package/dist/backend/build-info.json +2 -2
  40. package/dist/cli/backend/src/services/slack/slack-team-channel.service.d.ts +24 -0
  41. package/dist/cli/backend/src/services/slack/slack-team-channel.service.d.ts.map +1 -1
  42. package/dist/cli/backend/src/services/slack/slack-team-channel.service.js +60 -1
  43. package/dist/cli/backend/src/services/slack/slack-team-channel.service.js.map +1 -1
  44. package/dist/cli/backend/src/services/slack/slack.service.d.ts +17 -0
  45. package/dist/cli/backend/src/services/slack/slack.service.d.ts.map +1 -1
  46. package/dist/cli/backend/src/services/slack/slack.service.js +32 -0
  47. package/dist/cli/backend/src/services/slack/slack.service.js.map +1 -1
  48. package/dist/cli/backend/src/types/slack.types.d.ts +10 -0
  49. package/dist/cli/backend/src/types/slack.types.d.ts.map +1 -1
  50. package/dist/cli/backend/src/types/slack.types.js.map +1 -1
  51. package/dist/cli/backend/src/utils/incomplete-turn.utils.d.ts +1 -1
  52. package/dist/cli/backend/src/utils/incomplete-turn.utils.d.ts.map +1 -1
  53. package/dist/cli/backend/src/utils/incomplete-turn.utils.js +4 -0
  54. package/dist/cli/backend/src/utils/incomplete-turn.utils.js.map +1 -1
  55. package/package.json +1 -1
  56. package/packages/crewly-agent/src/eval/desktop/desktop-tasks.test.ts +96 -0
  57. package/packages/crewly-agent/src/eval/desktop/desktop-tasks.ts +226 -0
  58. package/packages/crewly-agent/src/runtime/agent-runner.service.ts +20 -1
  59. package/packages/crewly-agent/src/runtime/computer.tool.test.ts +219 -0
  60. package/packages/crewly-agent/src/runtime/computer.tool.ts +405 -0
  61. package/packages/crewly-agent/src/runtime/desktop-checkpoint.test.ts +136 -0
  62. package/packages/crewly-agent/src/runtime/desktop-checkpoint.ts +231 -0
  63. package/packages/crewly-agent/src/runtime/desktop-recovery.test.ts +100 -0
  64. package/packages/crewly-agent/src/runtime/desktop-recovery.ts +195 -0
  65. package/packages/crewly-agent/src/runtime/desktop-task-runtime.test.ts +251 -0
  66. package/packages/crewly-agent/src/runtime/desktop-task-runtime.ts +423 -0
  67. package/packages/crewly-agent/src/runtime/desktop-task.tool.test.ts +218 -0
  68. package/packages/crewly-agent/src/runtime/desktop-task.tool.ts +343 -0
  69. package/packages/crewly-agent/src/runtime/tool-registry.test.ts +17 -0
  70. package/packages/crewly-agent/src/runtime/tool-registry.ts +54 -0
  71. package/packages/crewly-agent/src/runtime/types.ts +10 -1
  72. package/config/skills/agent/vnc-browser/SKILL.md +0 -140
@@ -220,7 +220,16 @@ export type IncompleteReason =
220
220
  /** Hit `maxSteps` with work still outstanding. */
221
221
  | 'steps-exhausted'
222
222
  /** Provider refused on content grounds. */
223
- | 'content-filter';
223
+ | 'content-filter'
224
+ /**
225
+ * A desktop task ended with subgoals whose checkpoints never held.
226
+ *
227
+ * The turn itself may have finished perfectly well — the model stopped when
228
+ * it meant to. That is precisely the case worth catching: an agent that
229
+ * believes it saved the file and says so is indistinguishable, from the
230
+ * outside, from one that did.
231
+ */
232
+ | 'desktop-unverified';
224
233
 
225
234
  /** Describes a turn that ended early. */
226
235
  export interface IncompleteRun {
@@ -1,140 +0,0 @@
1
- ---
2
- name: VNC Remote Browser Access
3
- description: Launch a virtual display with VNC + noVNC + cloudflared for remote browser viewing and control. Enables human-in-the-loop login/verification when Playwright needs manual interaction.
4
- version: 1.0.0
5
- category: browser
6
- skillType: claude-skill
7
- assignableRoles:
8
- - developer
9
- - qa
10
- - qa-engineer
11
- - fullstack-dev
12
- - backend-developer
13
- - frontend-developer
14
- - generalist
15
- triggers:
16
- - vnc
17
- - remote browser
18
- - browser login
19
- - manual login
20
- - human verification
21
- - captcha
22
- - 2fa
23
- - playwright headful
24
- tags:
25
- - vnc
26
- - browser
27
- - remote-access
28
- - playwright
29
- - noVNC
30
- - cloudflared
31
- execution:
32
- type: script
33
- script:
34
- file: execute.sh
35
- interpreter: bash
36
- timeoutMs: 60000
37
- ---
38
-
39
- # VNC Remote Browser Access
40
-
41
- Bridge macOS Screen Sharing to a public HTTPS URL via noVNC + cloudflared. This lets humans remotely view and control the Mac desktop (and any browser on it) from anywhere — essential when Playwright hits a login page, CAPTCHA, or 2FA prompt.
42
-
43
- The stack: **macOS Screen Sharing** (built-in VNC, port 5900) → **websockify + noVNC** (web client, port 6080) → **cloudflared** (public HTTPS tunnel).
44
-
45
- ## Prerequisites
46
-
47
- **macOS Screen Sharing must be enabled manually** before using this skill:
48
-
49
- 1. Open **System Settings** → **General** → **Sharing**
50
- 2. Turn on **Screen Sharing**
51
- 3. (Optional) Set a VNC password under Screen Sharing options
52
-
53
- This only needs to be done once. Screen Sharing runs as a system service and persists across reboots.
54
-
55
- ## When to Use
56
-
57
- - Playwright hits a login page that requires human credentials
58
- - A website shows a CAPTCHA or anti-bot challenge
59
- - Two-factor authentication (2FA/MFA) requires a code from the user
60
- - You need the user to visually verify something in the browser
61
- - Any browser automation step that needs human-in-the-loop interaction
62
-
63
- ## Parameters
64
-
65
- | Parameter | Required | Description |
66
- |-----------|----------|-------------|
67
- | `action` | Yes | One of: `start`, `stop`, `status`, `get-url` |
68
-
69
- ### Actions
70
-
71
- - **start** — Check Screen Sharing is on, install deps if needed, launch websockify + noVNC + cloudflared. Returns the public URL.
72
- - **stop** — Shut down websockify + cloudflared. Does NOT touch Screen Sharing.
73
- - **status** — Check Screen Sharing (port 5900), websockify, and cloudflared status.
74
- - **get-url** — Retrieve the cloudflared public URL.
75
-
76
- ## Examples
77
-
78
- ### Start and get the public URL
79
-
80
- ```bash
81
- bash config/skills/agent/vnc-browser/execute.sh '{"action":"start"}'
82
- ```
83
-
84
- Output:
85
- ```json
86
- {
87
- "success": true,
88
- "status": "started",
89
- "publicUrl": "https://abc-xyz.trycloudflare.com/vnc.html?autoconnect=true",
90
- "localUrl": "http://localhost:6080/vnc.html?autoconnect=true",
91
- "hint": "Share the publicUrl with the user..."
92
- }
93
- ```
94
-
95
- ### Send the URL to the user via Slack
96
-
97
- ```bash
98
- URL=$(bash config/skills/agent/vnc-browser/execute.sh '{"action":"get-url"}' | jq -r '.publicUrl')
99
- # Then send via reply-slack or report-status
100
- ```
101
-
102
- ### Check status
103
-
104
- ```bash
105
- bash config/skills/agent/vnc-browser/execute.sh '{"action":"status"}'
106
- ```
107
-
108
- ### Stop when done
109
-
110
- ```bash
111
- bash config/skills/agent/vnc-browser/execute.sh '{"action":"stop"}'
112
- ```
113
-
114
- ## Workflow
115
-
116
- 1. **Ensure Screen Sharing is on** (one-time setup)
117
- 2. **Start VNC bridge** → get the public URL
118
- 3. **Launch Playwright** in headful mode (the browser appears on the Mac desktop)
119
- 4. **Navigate** to the page requiring human interaction
120
- 5. **Send the public URL** to the user via Slack
121
- 6. **Wait** for the user to complete the manual step (login, CAPTCHA, etc.)
122
- 7. **Continue automation** once the browser is past the manual step
123
- 8. **Stop VNC bridge** when no longer needed
124
-
125
- ## Technical Details
126
-
127
- - **VNC source**: macOS Screen Sharing on port 5900 (system-managed)
128
- - **Web client**: noVNC on port 6080 via websockify
129
- - **Public tunnel**: cloudflared Quick Tunnel (random `*.trycloudflare.com` subdomain)
130
- - **Security**: Screen Sharing requires macOS user credentials; cloudflared provides HTTPS
131
- - **Dependencies**: websockify (pip), noVNC (GitHub release), cloudflared (Homebrew) — auto-installed on first run
132
- - **PID files**: `~/.crewly/vnc/` for websockify and cloudflared lifecycle
133
- - **Note**: This skill does NOT start or stop macOS Screen Sharing — that is user-managed
134
-
135
- ## Requirements
136
-
137
- - macOS with Screen Sharing enabled
138
- - Homebrew installed
139
- - Python 3 with pip (for websockify)
140
- - Internet access (for cloudflared tunnel and dependency installation)