crewly 1.20.40 → 1.20.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/config/skills/_common/desktop-guards.sh +485 -0
  2. package/config/skills/_common/desktop-guards.test.sh +242 -0
  3. package/config/skills/_common/desktop-perceive.swift +530 -0
  4. package/config/skills/_common/desktop-presence.swift +343 -0
  5. package/config/skills/agent/_common/desktop-guards.sh +4 -0
  6. package/config/skills/agent/computer-use/SKILL.md +88 -0
  7. package/config/skills/agent/computer-use/execute.sh +249 -3
  8. package/config/skills/agent/desktop-app-control/SKILL.md +19 -0
  9. package/config/skills/agent/remote-browser/SKILL.md +19 -0
  10. package/dist/backend/backend/src/controllers/desktop/desktop.controller.d.ts +105 -0
  11. package/dist/backend/backend/src/controllers/desktop/desktop.controller.d.ts.map +1 -0
  12. package/dist/backend/backend/src/controllers/desktop/desktop.controller.js +278 -0
  13. package/dist/backend/backend/src/controllers/desktop/desktop.controller.js.map +1 -0
  14. package/dist/backend/backend/src/controllers/desktop/desktop.routes.d.ts +21 -0
  15. package/dist/backend/backend/src/controllers/desktop/desktop.routes.d.ts.map +1 -0
  16. package/dist/backend/backend/src/controllers/desktop/desktop.routes.js +31 -0
  17. package/dist/backend/backend/src/controllers/desktop/desktop.routes.js.map +1 -0
  18. package/dist/backend/backend/src/routes/api.routes.d.ts.map +1 -1
  19. package/dist/backend/backend/src/routes/api.routes.js +3 -0
  20. package/dist/backend/backend/src/routes/api.routes.js.map +1 -1
  21. package/dist/backend/backend/src/services/cloud/mobile-api-relay.service.d.ts.map +1 -1
  22. package/dist/backend/backend/src/services/cloud/mobile-api-relay.service.js +9 -0
  23. package/dist/backend/backend/src/services/cloud/mobile-api-relay.service.js.map +1 -1
  24. package/dist/backend/backend/src/services/slack/slack-team-channel.service.d.ts +24 -0
  25. package/dist/backend/backend/src/services/slack/slack-team-channel.service.d.ts.map +1 -1
  26. package/dist/backend/backend/src/services/slack/slack-team-channel.service.js +60 -1
  27. package/dist/backend/backend/src/services/slack/slack-team-channel.service.js.map +1 -1
  28. package/dist/backend/backend/src/services/slack/slack.service.d.ts +17 -0
  29. package/dist/backend/backend/src/services/slack/slack.service.d.ts.map +1 -1
  30. package/dist/backend/backend/src/services/slack/slack.service.js +32 -0
  31. package/dist/backend/backend/src/services/slack/slack.service.js.map +1 -1
  32. package/dist/backend/backend/src/types/slack.types.d.ts +10 -0
  33. package/dist/backend/backend/src/types/slack.types.d.ts.map +1 -1
  34. package/dist/backend/backend/src/types/slack.types.js.map +1 -1
  35. package/dist/backend/backend/src/utils/incomplete-turn.utils.d.ts +1 -1
  36. package/dist/backend/backend/src/utils/incomplete-turn.utils.d.ts.map +1 -1
  37. package/dist/backend/backend/src/utils/incomplete-turn.utils.js +4 -0
  38. package/dist/backend/backend/src/utils/incomplete-turn.utils.js.map +1 -1
  39. package/dist/backend/build-info.json +2 -2
  40. package/dist/cli/backend/src/services/slack/slack-team-channel.service.d.ts +24 -0
  41. package/dist/cli/backend/src/services/slack/slack-team-channel.service.d.ts.map +1 -1
  42. package/dist/cli/backend/src/services/slack/slack-team-channel.service.js +60 -1
  43. package/dist/cli/backend/src/services/slack/slack-team-channel.service.js.map +1 -1
  44. package/dist/cli/backend/src/services/slack/slack.service.d.ts +17 -0
  45. package/dist/cli/backend/src/services/slack/slack.service.d.ts.map +1 -1
  46. package/dist/cli/backend/src/services/slack/slack.service.js +32 -0
  47. package/dist/cli/backend/src/services/slack/slack.service.js.map +1 -1
  48. package/dist/cli/backend/src/types/slack.types.d.ts +10 -0
  49. package/dist/cli/backend/src/types/slack.types.d.ts.map +1 -1
  50. package/dist/cli/backend/src/types/slack.types.js.map +1 -1
  51. package/dist/cli/backend/src/utils/incomplete-turn.utils.d.ts +1 -1
  52. package/dist/cli/backend/src/utils/incomplete-turn.utils.d.ts.map +1 -1
  53. package/dist/cli/backend/src/utils/incomplete-turn.utils.js +4 -0
  54. package/dist/cli/backend/src/utils/incomplete-turn.utils.js.map +1 -1
  55. package/package.json +1 -1
  56. package/packages/crewly-agent/src/eval/desktop/desktop-tasks.test.ts +96 -0
  57. package/packages/crewly-agent/src/eval/desktop/desktop-tasks.ts +226 -0
  58. package/packages/crewly-agent/src/runtime/agent-runner.service.ts +20 -1
  59. package/packages/crewly-agent/src/runtime/computer.tool.test.ts +219 -0
  60. package/packages/crewly-agent/src/runtime/computer.tool.ts +405 -0
  61. package/packages/crewly-agent/src/runtime/desktop-checkpoint.test.ts +136 -0
  62. package/packages/crewly-agent/src/runtime/desktop-checkpoint.ts +231 -0
  63. package/packages/crewly-agent/src/runtime/desktop-recovery.test.ts +100 -0
  64. package/packages/crewly-agent/src/runtime/desktop-recovery.ts +195 -0
  65. package/packages/crewly-agent/src/runtime/desktop-task-runtime.test.ts +251 -0
  66. package/packages/crewly-agent/src/runtime/desktop-task-runtime.ts +423 -0
  67. package/packages/crewly-agent/src/runtime/desktop-task.tool.test.ts +218 -0
  68. package/packages/crewly-agent/src/runtime/desktop-task.tool.ts +343 -0
  69. package/packages/crewly-agent/src/runtime/tool-registry.test.ts +17 -0
  70. package/packages/crewly-agent/src/runtime/tool-registry.ts +54 -0
  71. package/packages/crewly-agent/src/runtime/types.ts +10 -1
  72. package/config/skills/agent/vnc-browser/SKILL.md +0 -140
@@ -0,0 +1,485 @@
1
+ #!/usr/bin/env bash
2
+ # =============================================================================
3
+ # desktop-guards.sh — safety rails shared by every skill that drives the
4
+ # owner's real mouse, keyboard and screen.
5
+ #
6
+ # Sourced rather than copied because there are already two divergent
7
+ # computer-use implementations in this repo (config/skills/agent/computer-use
8
+ # and .../marketplace/computer-use) with different action sets. Duplicating
9
+ # the rails into both is how they drift apart, and a guard that exists in only
10
+ # one copy is worse than none — it makes the skill look safe.
11
+ #
12
+ # Callers must have sourced _common/lib.sh first (for error_exit) and must set
13
+ # ACTION and INPUT before calling cu_apply_guards.
14
+ #
15
+ # Usage:
16
+ # source "${SCRIPT_DIR}/../../_common/desktop-guards.sh"
17
+ # cu_apply_guards
18
+ # =============================================================================
19
+
20
+
21
+ CREWLY_HOME_DIR="${CREWLY_HOME:-$HOME/.crewly}"
22
+ DESKTOP_LOCK="${CREWLY_HOME_DIR}/desktop.lock"
23
+ DESKTOP_LOG="${CREWLY_HOME_DIR}/desktop-actions.jsonl"
24
+ DESKTOP_STOP="${CREWLY_HOME_DIR}/desktop.stop"
25
+ LOCK_TTL_SECONDS="${CREWLY_DESKTOP_LOCK_TTL:-120}"
26
+ HOLDER="${CREWLY_SESSION_NAME:-unknown-agent}"
27
+
28
+ mkdir -p "$CREWLY_HOME_DIR" 2>/dev/null || true
29
+
30
+ # ---------------------------------------------------------------------------
31
+ # cu_fail reason message [extra_json]
32
+ #
33
+ # A refusal the agent can act on. Every guard answers in this one shape so a
34
+ # model does not have to parse prose to tell "you lack permission" from "that
35
+ # element was not found".
36
+ # ---------------------------------------------------------------------------
37
+ cu_fail() {
38
+ local reason="$1" message="$2" extra="${3:-{\}}"
39
+ jq -n --arg a "$ACTION" --arg r "$reason" --arg m "$message" --argjson x "$extra" \
40
+ '{success:false, action:$a, reason:$r, message:$m} + $x'
41
+ exit 1
42
+ }
43
+
44
+ # ---------------------------------------------------------------------------
45
+ # Permission preflight
46
+ #
47
+ # macOS grants TCC permissions to the *process that asks*, which for a skill is
48
+ # whatever launched the agent — Terminal, iTerm, or the crewly backend's node.
49
+ # So the answer names the process, otherwise the owner cannot tell what to tick.
50
+ # ---------------------------------------------------------------------------
51
+ asking_process() {
52
+ # The nearest ancestor the user would recognise in the TCC pane.
53
+ ps -o comm= -p "${PPID:-$$}" 2>/dev/null | sed 's:.*/::' || echo "your terminal"
54
+ }
55
+
56
+ has_screen_recording() {
57
+ # CGPreflightScreenCaptureAccess is a plain C function: importing CoreGraphics
58
+ # is not enough, JXA only sees it once it is bound explicitly.
59
+ osascript -l JavaScript -e '
60
+ ObjC.import("CoreGraphics");
61
+ ObjC.bindFunction("CGPreflightScreenCaptureAccess", ["bool", []]);
62
+ $.CGPreflightScreenCaptureAccess() ? "yes" : "no"
63
+ ' 2>/dev/null
64
+ }
65
+
66
+ has_accessibility() {
67
+ osascript -l JavaScript -e 'ObjC.import("ApplicationServices"); $.AXIsProcessTrusted() ? "yes" : "no"' 2>/dev/null
68
+ }
69
+
70
+ require_screen_recording() {
71
+ [ "$(has_screen_recording)" = "yes" ] && return 0
72
+ cu_fail "permission_required" \
73
+ "Screen Recording is not granted to $(asking_process), so every screenshot comes back blank. Grant it, then quit and reopen that app — macOS only applies the change on restart." \
74
+ "$(jq -n --arg p "$(asking_process)" '{permission:"screen-recording", grantTo:$p, howTo:"System Settings → Privacy & Security → Screen & System Audio Recording"}')"
75
+ }
76
+
77
+ require_accessibility() {
78
+ [ "$(has_accessibility)" = "yes" ] && return 0
79
+ cu_fail "permission_required" \
80
+ "Accessibility is not granted to $(asking_process), so clicks and keystrokes are silently discarded. Grant it, then quit and reopen that app." \
81
+ "$(jq -n --arg p "$(asking_process)" '{permission:"accessibility", grantTo:$p, howTo:"System Settings → Privacy & Security → Accessibility"}')"
82
+ }
83
+
84
+ # ---------------------------------------------------------------------------
85
+ # Stop switch
86
+ #
87
+ # A file, so anything can set it: the owner, a hotkey, the backend. Checked
88
+ # before every action rather than only at the start, because the point is to
89
+ # stop an agent that is already mid-task.
90
+ # ---------------------------------------------------------------------------
91
+ require_not_stopped() {
92
+ [ -f "$DESKTOP_STOP" ] || return 0
93
+ cu_fail "stopped_by_user" \
94
+ "Desktop control is stopped. The owner halted it; it stays off until $DESKTOP_STOP is removed." \
95
+ "$(jq -n --arg f "$DESKTOP_STOP" '{stopFile:$f}')"
96
+ }
97
+
98
+ # ---------------------------------------------------------------------------
99
+ # Platform
100
+ #
101
+ # Everything here is macOS: screencapture, AppleScript, CoreGraphics event
102
+ # taps, the Accessibility API. On Linux or Windows the commands are simply
103
+ # absent, and the failure an agent saw was `osascript: command not found` —
104
+ # which reads as a broken install rather than an unsupported platform, so it
105
+ # retried. Saying so plainly is the whole fix until the Linux backend of
106
+ # Phase 6 exists.
107
+ # ---------------------------------------------------------------------------
108
+ require_macos() {
109
+ [ "$(uname -s)" = "Darwin" ] && return 0
110
+ cu_fail "unsupported_platform" \
111
+ "Desktop control is macOS-only for now. On $(uname -s) there is no screen to drive from here — use the browser tools for web work, or a skill for anything with an API." \
112
+ "$(jq -n --arg p "$(uname -s)" '{platform:$p, supported:["Darwin"]}')"
113
+ }
114
+
115
+ # ---------------------------------------------------------------------------
116
+ # Locked screen
117
+ #
118
+ # Accessibility does not fail while the screen is locked — it answers with
119
+ # rubbish. Every window of every app comes back with role AXApplication and no
120
+ # real content, and System Events cannot even name the frontmost process. An
121
+ # agent reading that sees a plausible-looking tree and acts on it, clicking
122
+ # coordinates that belong to nothing (2026-09-20, found while testing the
123
+ # perception layer against a locked Mac).
124
+ #
125
+ # There is also nothing useful to do on a locked screen, so this is a refusal
126
+ # rather than a warning.
127
+ # ---------------------------------------------------------------------------
128
+ screen_is_locked() {
129
+ osascript -l JavaScript -e '
130
+ ObjC.import("CoreGraphics");
131
+ ObjC.bindFunction("CGSessionCopyCurrentDictionary", ["id", []]);
132
+ var d = $.CGSessionCopyCurrentDictionary();
133
+ d && ObjC.unwrap(d.objectForKey("CGSSessionScreenIsLocked")) ? "yes" : "no"
134
+ ' 2>/dev/null
135
+ }
136
+
137
+ require_unlocked() {
138
+ [ "$(screen_is_locked)" = "yes" ] || return 0
139
+ cu_fail "screen_locked" \
140
+ "The screen is locked. Accessibility answers with placeholder data while it is, so anything read now would be wrong and anything clicked would land on nothing. Wait for the owner to unlock, or ask them to." \
141
+ '{"recoverable":true}'
142
+ }
143
+
144
+ # ---------------------------------------------------------------------------
145
+ # Mutual exclusion
146
+ #
147
+ # One machine has one mouse and one keyboard focus, so two agents acting at
148
+ # once corrupt each other. The lock carries its holder and an expiry, so a
149
+ # crashed agent cannot wedge the desktop forever.
150
+ # ---------------------------------------------------------------------------
151
+ acquire_desktop_lock() {
152
+ local now holder_existing expires
153
+ now=$(date +%s)
154
+ if [ -f "$DESKTOP_LOCK" ]; then
155
+ holder_existing=$(jq -r '.holder // "unknown"' "$DESKTOP_LOCK" 2>/dev/null || echo unknown)
156
+ expires=$(jq -r '.expiresAt // 0' "$DESKTOP_LOCK" 2>/dev/null || echo 0)
157
+ if [ "$holder_existing" != "$HOLDER" ] && [ "$expires" -gt "$now" ] 2>/dev/null; then
158
+ cu_fail "desktop_busy" \
159
+ "$holder_existing is using the desktop until $(date -r "$expires" '+%H:%M:%S'). Wait, or ask that agent to finish." \
160
+ "$(jq -n --arg h "$holder_existing" --argjson e "$expires" '{heldBy:$h, expiresAt:$e}')"
161
+ fi
162
+ fi
163
+ jq -n --arg h "$HOLDER" --argjson e "$((now + LOCK_TTL_SECONDS))" --arg a "$ACTION" \
164
+ '{holder:$h, expiresAt:$e, lastAction:$a}' > "$DESKTOP_LOCK" 2>/dev/null || true
165
+ }
166
+
167
+ # ---------------------------------------------------------------------------
168
+ # Audit
169
+ #
170
+ # Appended before the action runs, so an action that hangs or crashes the shell
171
+ # still leaves a trace. Screenshots of each step come later (Phase 5).
172
+ # ---------------------------------------------------------------------------
173
+ DESKTOP_SHOTS="${CREWLY_HOME_DIR}/desktop-actions"
174
+
175
+ # Small enough that a day of them costs a few megabytes, large enough to see
176
+ # which window was in front and whether a dialog was open.
177
+ THUMB_WIDTH="${CREWLY_DESKTOP_THUMB_WIDTH:-480}"
178
+
179
+ # capture_thumb <label> → path, or empty when it could not be taken
180
+ #
181
+ # Failure is silent and empty: an action must never fail because its evidence
182
+ # could not be recorded.
183
+ capture_thumb() {
184
+ [ "${CREWLY_DESKTOP_AUDIT_SHOTS:-1}" = "1" ] || return 0
185
+ case "$ACTION" in
186
+ click|move|type|key|scroll|drag|focus|focus-app|open-url|click-text|click-ref|fill-ref) ;;
187
+ # Handing over to a person: the picture of what the agent is stuck on is
188
+ # the most useful thing the owner gets, especially on a phone.
189
+ request-human) ;;
190
+ *) return 0 ;;
191
+ esac
192
+ local day dir file
193
+ day=$(date -u +%Y-%m-%d)
194
+ dir="${DESKTOP_SHOTS}/${day}"
195
+ mkdir -p "$dir" 2>/dev/null || return 0
196
+ file="${dir}/$(date -u +%H%M%S)-$$-$1.jpg"
197
+ screencapture -x -t jpg "$file" 2>/dev/null || return 0
198
+ sips --resampleWidth "$THUMB_WIDTH" "$file" --out "$file" >/dev/null 2>&1 || true
199
+ printf '%s' "$file"
200
+ }
201
+
202
+ log_action() {
203
+ local before
204
+ before=$(capture_thumb before)
205
+ jq -nc --arg t "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg s "$HOLDER" --arg a "$ACTION" \
206
+ --argjson i "$INPUT" --arg b "$before" \
207
+ '{at:$t, session:$s, action:$a, input:$i} + (if $b == "" then {} else {before:$b} end)' \
208
+ >> "$DESKTOP_LOG" 2>/dev/null || true
209
+ # The "after" shot is taken on the way out, once the action has landed, so
210
+ # the pair shows cause and effect rather than two pictures of the same
211
+ # moment. Only for actions that got past every rail — a refusal changed
212
+ # nothing and a second identical picture is just noise.
213
+ CU_LOG_BEFORE="$before"
214
+ }
215
+
216
+ # cu_log_after — called once the action has run.
217
+ cu_log_after() {
218
+ [ -n "${CU_LOG_BEFORE:-}" ] || return 0
219
+ local after
220
+ # Give the UI a moment to actually change; without it the pair is useless.
221
+ sleep 0.35
222
+ after=$(capture_thumb after)
223
+ [ -n "$after" ] || return 0
224
+ jq -nc --arg t "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg s "$HOLDER" --arg a "$ACTION" \
225
+ --arg b "$CU_LOG_BEFORE" --arg f "$after" \
226
+ '{at:$t, session:$s, action:$a, phase:"after", before:$b, after:$f}' \
227
+ >> "$DESKTOP_LOG" 2>/dev/null || true
228
+ }
229
+
230
+ # ---------------------------------------------------------------------------
231
+ # Destructive key combos
232
+ #
233
+ # Blocked outright rather than sent for approval: these are almost never what
234
+ # an automation legitimately wants, and a blocked action the agent can route
235
+ # around beats a prompt the owner learns to click through. An operator who
236
+ # really means it sets CREWLY_DESKTOP_ALLOW_DESTRUCTIVE=1 for that one run.
237
+ # ---------------------------------------------------------------------------
238
+ DESTRUCTIVE_KEYS='^(command|cmd)\+(q|w)$|^(command|cmd)\+(delete|backspace)$|^(command|cmd)\+shift\+(delete|backspace)$|^(command|cmd)\+(option|alt)\+escape$'
239
+
240
+ guard_destructive_key() {
241
+ local key_name="$1" normalized
242
+ normalized=$(printf '%s' "$key_name" | tr '[:upper:]' '[:lower:]' | tr -d ' ')
243
+ printf '%s' "$normalized" | grep -qE "$DESTRUCTIVE_KEYS" || return 0
244
+ [ "${CREWLY_DESKTOP_ALLOW_DESTRUCTIVE:-}" = "1" ] && return 0
245
+ cu_fail "destructive_blocked" \
246
+ "\"$key_name\" quits an app or deletes data, which desktop control refuses by default. Achieve the goal another way, or ask the owner to re-run with CREWLY_DESKTOP_ALLOW_DESTRUCTIVE=1." \
247
+ "$(jq -n --arg k "$key_name" '{key:$k, override:"CREWLY_DESKTOP_ALLOW_DESTRUCTIVE=1"}')"
248
+ }
249
+
250
+ # ---------------------------------------------------------------------------
251
+ # Apps that are never automated
252
+ #
253
+ # Typing into a password manager or the Keychain is indistinguishable from
254
+ # exfiltrating credentials, so focus is refused rather than audited.
255
+ # ---------------------------------------------------------------------------
256
+ DENIED_APPS='^(system settings|system preferences|keychain access|1password|1password 7|1password 8|bitwarden|lastpass|dashlane|passwords)$'
257
+
258
+ guard_denied_app() {
259
+ local app_name="$1" normalized
260
+ normalized=$(printf '%s' "$app_name" | tr '[:upper:]' '[:lower:]')
261
+ printf '%s' "$normalized" | grep -qE "$DENIED_APPS" || return 0
262
+ cu_fail "app_not_allowed" \
263
+ "Desktop control does not drive $app_name — it holds the owner's credentials. Ask the owner to do this part." \
264
+ "$(jq -n --arg a "$app_name" '{app:$a}')"
265
+ }
266
+
267
+ # ---------------------------------------------------------------------------
268
+ # Secure text fields
269
+ #
270
+ # `type` goes to whatever has focus, so without this a password box is just
271
+ # another text box. Fails open when the focused element cannot be read: AX
272
+ # gaps are common and refusing every unreadable field would break normal use.
273
+ # ---------------------------------------------------------------------------
274
+ focused_role() {
275
+ osascript -l JavaScript -e '
276
+ ObjC.import("ApplicationServices");
277
+ function roleOf() {
278
+ var sys = $.AXUIElementCreateSystemWide();
279
+ var appRef = Ref();
280
+ if ($.AXUIElementCopyAttributeValue(sys, $.CFSTR("AXFocusedApplication"), appRef) !== 0) return "";
281
+ var elemRef = Ref();
282
+ if ($.AXUIElementCopyAttributeValue(appRef[0], $.CFSTR("AXFocusedUIElement"), elemRef) !== 0) return "";
283
+ var roleRef = Ref();
284
+ if ($.AXUIElementCopyAttributeValue(elemRef[0], $.CFSTR("AXRole"), roleRef) !== 0) return "";
285
+ return ObjC.unwrap(roleRef[0]) || "";
286
+ }
287
+ roleOf();
288
+ ' 2>/dev/null || echo ""
289
+ }
290
+
291
+ guard_secure_input() {
292
+ local role
293
+ role=$(focused_role)
294
+ [ "$role" = "AXSecureTextField" ] || return 0
295
+ cu_fail "secure_field" \
296
+ "The focused field is a password box. Desktop control never types into one — ask the owner to enter it." \
297
+ '{"focusedRole":"AXSecureTextField"}'
298
+ }
299
+
300
+ # ---------------------------------------------------------------------------
301
+ # check-permissions — report both TCC grants without performing any action.
302
+ # ---------------------------------------------------------------------------
303
+ do_check_permissions() {
304
+ local screen ax proc
305
+ screen=$(has_screen_recording); ax=$(has_accessibility); proc=$(asking_process)
306
+ jq -n --arg p "$proc" \
307
+ --argjson s "$([ "$screen" = yes ] && echo true || echo false)" \
308
+ --argjson a "$([ "$ax" = yes ] && echo true || echo false)" \
309
+ '{success:true, action:"check-permissions", askingProcess:$p,
310
+ screenRecording:$s, accessibility:$a,
311
+ ready:($s and $a),
312
+ howTo:"System Settings → Privacy & Security → (Screen & System Audio Recording | Accessibility). Grant to the process named above, then quit and reopen it."}'
313
+ }
314
+
315
+ # ---------------------------------------------------------------------------
316
+ # The perception helper
317
+ #
318
+ # A small Swift binary (AX tree, Vision OCR, display list). Compiled on first
319
+ # use and cached, because shipping a binary in a skill directory means
320
+ # shipping an unsigned one a user cannot verify, and because the source is
321
+ # the thing worth reviewing. Rebuilt whenever the source is newer.
322
+ #
323
+ # Swift rather than JXA: the scripting bridge costs a round trip per
324
+ # attribute, so reading one window took seconds — too slow to do before every
325
+ # action. Raw AXUIElement does it in tens of milliseconds.
326
+ # ---------------------------------------------------------------------------
327
+ PERCEIVE_SRC="${CREWLY_SKILLS_COMMON:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}/desktop-perceive.swift"
328
+ PERCEIVE_BIN="${CREWLY_HOME_DIR}/bin/desktop-perceive"
329
+
330
+ cu_perceive() {
331
+ if [ ! -x "$PERCEIVE_BIN" ] || [ "$PERCEIVE_SRC" -nt "$PERCEIVE_BIN" ]; then
332
+ if ! command -v swiftc >/dev/null 2>&1; then
333
+ cu_fail "toolchain_missing" \
334
+ "Element-level perception needs swiftc, which comes with the Xcode Command Line Tools. Install them with: xcode-select --install" \
335
+ '{"install":"xcode-select --install"}'
336
+ fi
337
+ mkdir -p "$(dirname "$PERCEIVE_BIN")"
338
+ if ! swiftc -O -o "$PERCEIVE_BIN" "$PERCEIVE_SRC" 2>"${CREWLY_HOME_DIR}/desktop-perceive-build.log"; then
339
+ cu_fail "build_failed" \
340
+ "Could not build the perception helper. See ${CREWLY_HOME_DIR}/desktop-perceive-build.log." \
341
+ "$(jq -n --arg l "${CREWLY_HOME_DIR}/desktop-perceive-build.log" '{log:$l}')"
342
+ fi
343
+ fi
344
+ "$PERCEIVE_BIN" "$@"
345
+ }
346
+
347
+ # ---------------------------------------------------------------------------
348
+ # Presence: the banner, the hotkey, and the eye on the owner's own input
349
+ #
350
+ # The browser line has shown a takeover banner since April; the desktop showed
351
+ # nothing, and a pointer that starts moving by itself is the difference
352
+ # between automation and a haunting. The resident process owns no policy — it
353
+ # writes the same desktop.stop / desktop.pause files these rails already read,
354
+ # so it can die without leaving anything un-enforced.
355
+ #
356
+ # Started on demand and refreshed before each action; it exits by itself once
357
+ # the refreshes stop, so a crashed agent cannot leave a banner claiming to be
358
+ # working.
359
+ # ---------------------------------------------------------------------------
360
+ PRESENCE_SRC="${CREWLY_SKILLS_COMMON:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}/desktop-presence.swift"
361
+ PRESENCE_BIN="${CREWLY_HOME_DIR}/bin/desktop-presence"
362
+ DESKTOP_PAUSE="${CREWLY_HOME_DIR}/desktop.pause"
363
+
364
+ cu_presence() {
365
+ # Never fatal: an agent that cannot show a banner should still be stoppable
366
+ # by the rails, and failing the action would be worse than a missing banner.
367
+ [ "${CREWLY_DESKTOP_NO_BANNER:-}" = "1" ] && return 0
368
+ if [ ! -x "$PRESENCE_BIN" ] || [ "$PRESENCE_SRC" -nt "$PRESENCE_BIN" ]; then
369
+ command -v swiftc >/dev/null 2>&1 || return 0
370
+ mkdir -p "$(dirname "$PRESENCE_BIN")"
371
+ swiftc -O -o "$PRESENCE_BIN" "$PRESENCE_SRC" 2>"${CREWLY_HOME_DIR}/desktop-presence-build.log" || return 0
372
+ fi
373
+ "$PRESENCE_BIN" "$@" >/dev/null 2>&1 || true
374
+ }
375
+
376
+ # Show (or refresh) the banner for this action.
377
+ cu_presence_refresh() {
378
+ local goal="${CREWLY_AGENT_GOAL:-$ACTION}"
379
+ cu_presence begin --agent "$HOLDER" --goal "$goal"
380
+ # The window only exists while a foreground process is running; start one
381
+ # if none is.
382
+ pgrep -f "desktop-presence begin .*--foreground" >/dev/null 2>&1 || {
383
+ nohup "$PRESENCE_BIN" begin --agent "$HOLDER" --goal "$goal" --foreground >/dev/null 2>&1 &
384
+ }
385
+ }
386
+
387
+ # ---------------------------------------------------------------------------
388
+ # Paused
389
+ #
390
+ # Distinct from stopped: the owner reached for the mouse, or pressed Pause.
391
+ # The task is not abandoned — it waits, and the same banner resumes it. A
392
+ # refusal rather than a block, because a blocked shell holds the desktop lock
393
+ # and would stop the owner's own agents too.
394
+ # ---------------------------------------------------------------------------
395
+ require_not_paused() {
396
+ [ -f "$DESKTOP_PAUSE" ] || return 0
397
+ cu_fail "paused" \
398
+ "Desktop control is paused — the owner is using the machine. Wait, and try again when they hand it back; the task is not cancelled." \
399
+ '{"recoverable":true}'
400
+ }
401
+
402
+ # ---------------------------------------------------------------------------
403
+ # cu_apply_guards
404
+ #
405
+ # Run every rail that applies to $ACTION. Call once, before dispatch.
406
+ #
407
+ # Action names differ between the two computer-use forks (`focus` vs
408
+ # `focus-app`, and only one has `key`), so the lists below accept both; an
409
+ # action a fork does not have simply never matches.
410
+ # ---------------------------------------------------------------------------
411
+ cu_apply_guards() {
412
+ # Platform first: on the wrong OS not even "check what permissions I have"
413
+ # has an answer, and `osascript: command not found` reads as a broken
414
+ # install rather than an unsupported platform.
415
+ require_macos
416
+ [ "$ACTION" = "check-permissions" ] && return 0
417
+ [ "$ACTION" = "check-accessibility" ] && return 0
418
+ # Listing screens reads no window and touches nothing.
419
+ [ "$ACTION" = "displays" ] && return 0
420
+ # Asking for a person is how an agent gets *out* of being stuck, so it must
421
+ # work while paused — refusing it would leave the agent with nothing to do
422
+ # but retry the thing it already cannot do.
423
+ [ "$ACTION" = "request-human" ] && { require_macos; log_action; return 0; }
424
+
425
+ require_not_stopped
426
+ log_action
427
+
428
+ # Permanent policy first, before anything transient. Both "you may not press
429
+ # ⌘Q" and "the screen is locked" can be true at once, and answering with the
430
+ # transient one invites the agent to wait and retry something that will never
431
+ # be allowed. These are pure string checks, so they also work on a locked
432
+ # screen where reading the UI would not.
433
+ case "$ACTION" in
434
+ key) guard_destructive_key "$(printf '%s' "$INPUT" | jq -r '.key // empty')" ;;
435
+ focus|focus-app|open-url) guard_denied_app "$(printf '%s' "$INPUT" | jq -r '.app // empty')" ;;
436
+ esac
437
+
438
+ # Then the transient conditions, cheapest first.
439
+ require_not_paused
440
+ require_unlocked
441
+
442
+ case "$ACTION" in
443
+ screenshot|find|click-text|ocr) require_screen_recording ;;
444
+ esac
445
+ case "$ACTION" in
446
+ click|move|type|key|scroll|drag|focus|focus-app|open-url|list-apps|click-text|read-ui|get-text|\
447
+ snapshot|click-ref|fill-ref|resolve|wait-for)
448
+ require_accessibility ;;
449
+ esac
450
+
451
+ # Reads the focused element, so it has to come after the lock and permission
452
+ # checks — on a locked screen the answer would be meaningless.
453
+ # fill-ref's own secure-field refusal lives in the perception helper, which
454
+ # reads the target element's role directly instead of guessing from focus.
455
+ case "$ACTION" in
456
+ type) guard_secure_input ;;
457
+ esac
458
+
459
+ # Only actions that change something take the lock. Looking (snapshot, ocr,
460
+ # resolve, wait-for) must stay free: an agent waiting its turn still needs
461
+ # to see, and two agents reading at once cannot corrupt anything.
462
+ case "$ACTION" in
463
+ click|move|type|key|scroll|drag|focus|focus-app|open-url|click-text|click-ref|fill-ref)
464
+ acquire_desktop_lock ;;
465
+ esac
466
+
467
+ # Dry run: every rail above has passed, so the action *would* be allowed —
468
+ # report that and stop before touching the mouse. Exists because the rails
469
+ # can only be tested honestly by also testing what they let through, and a
470
+ # test suite must not open apps or send keystrokes on the owner's machine.
471
+ # Agents can use it the same way, to check an action before committing to it.
472
+ if [ "${CREWLY_DESKTOP_DRY_RUN:-}" = "1" ]; then
473
+ jq -n --arg a "$ACTION" '{success:true, action:$a, dryRun:true, wouldRun:true}'
474
+ exit 0
475
+ fi
476
+
477
+ # The banner goes up last, and only for actions that will actually move
478
+ # something. Reading the screen is not taking the machine over, and a dry
479
+ # run moves nothing at all — raising it for either would cry wolf, and a
480
+ # banner the owner learns to ignore is worse than none.
481
+ case "$ACTION" in
482
+ click|move|type|key|scroll|drag|focus|focus-app|open-url|click-text|click-ref|fill-ref)
483
+ cu_presence_refresh ;;
484
+ esac
485
+ }