crewly 1.20.35 → 1.20.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/skills/_common/desktop-guards.sh +485 -0
- package/config/skills/_common/desktop-guards.test.sh +242 -0
- package/config/skills/_common/desktop-perceive.swift +530 -0
- package/config/skills/_common/desktop-presence.swift +343 -0
- package/config/skills/_common/lib.sh +6 -0
- package/config/skills/agent/_common/desktop-guards.sh +4 -0
- package/config/skills/agent/computer-use/SKILL.md +88 -0
- package/config/skills/agent/computer-use/execute.sh +249 -3
- package/config/skills/agent/core/calendar-create/SKILL.md +10 -0
- package/config/skills/agent/core/calendar-create/execute.sh +6 -0
- package/config/skills/agent/core/calendar-list/SKILL.md +10 -0
- package/config/skills/agent/core/calendar-list/execute.sh +6 -0
- package/config/skills/agent/core/docs-read/SKILL.md +10 -0
- package/config/skills/agent/core/docs-read/execute.sh +6 -1
- package/config/skills/agent/core/docs-write/SKILL.md +10 -0
- package/config/skills/agent/core/docs-write/execute.sh +6 -1
- package/config/skills/agent/core/drive-read/SKILL.md +10 -0
- package/config/skills/agent/core/drive-read/execute.sh +6 -1
- package/config/skills/agent/core/drive-search/SKILL.md +10 -0
- package/config/skills/agent/core/drive-search/execute.sh +6 -1
- package/config/skills/agent/core/drive-upload/SKILL.md +10 -0
- package/config/skills/agent/core/drive-upload/execute.sh +6 -1
- package/config/skills/agent/core/gmail-read/SKILL.md +10 -0
- package/config/skills/agent/core/gmail-read/execute.sh +6 -0
- package/config/skills/agent/core/gmail-search/SKILL.md +10 -0
- package/config/skills/agent/core/gmail-search/execute.sh +6 -0
- package/config/skills/agent/core/gmail-send/SKILL.md +10 -0
- package/config/skills/agent/core/gmail-send/execute.sh +6 -0
- package/config/skills/agent/core/sheets-read/SKILL.md +10 -0
- package/config/skills/agent/core/sheets-read/execute.sh +6 -1
- package/config/skills/agent/core/sheets-write/SKILL.md +10 -0
- package/config/skills/agent/core/sheets-write/execute.sh +6 -1
- package/config/skills/agent/core/slides-create/SKILL.md +10 -0
- package/config/skills/agent/core/slides-create/execute.sh +6 -1
- package/config/skills/agent/core/slides-read/SKILL.md +10 -0
- package/config/skills/agent/core/slides-read/execute.sh +6 -1
- package/config/skills/agent/desktop-app-control/SKILL.md +19 -0
- package/config/skills/agent/remote-browser/SKILL.md +19 -0
- package/config/slack-app-manifest.json +16 -9
- package/dist/backend/backend/src/constants.d.ts +18 -4
- package/dist/backend/backend/src/constants.d.ts.map +1 -1
- package/dist/backend/backend/src/constants.js +16 -4
- package/dist/backend/backend/src/constants.js.map +1 -1
- package/dist/backend/backend/src/controllers/desktop/desktop.controller.d.ts +105 -0
- package/dist/backend/backend/src/controllers/desktop/desktop.controller.d.ts.map +1 -0
- package/dist/backend/backend/src/controllers/desktop/desktop.controller.js +278 -0
- package/dist/backend/backend/src/controllers/desktop/desktop.controller.js.map +1 -0
- package/dist/backend/backend/src/controllers/desktop/desktop.routes.d.ts +21 -0
- package/dist/backend/backend/src/controllers/desktop/desktop.routes.d.ts.map +1 -0
- package/dist/backend/backend/src/controllers/desktop/desktop.routes.js +31 -0
- package/dist/backend/backend/src/controllers/desktop/desktop.routes.js.map +1 -0
- package/dist/backend/backend/src/controllers/google/google.controller.d.ts +8 -0
- package/dist/backend/backend/src/controllers/google/google.controller.d.ts.map +1 -1
- package/dist/backend/backend/src/controllers/google/google.controller.js +137 -37
- package/dist/backend/backend/src/controllers/google/google.controller.js.map +1 -1
- package/dist/backend/backend/src/controllers/google/google.routes.d.ts +2 -1
- package/dist/backend/backend/src/controllers/google/google.routes.d.ts.map +1 -1
- package/dist/backend/backend/src/controllers/google/google.routes.js +4 -2
- package/dist/backend/backend/src/controllers/google/google.routes.js.map +1 -1
- package/dist/backend/backend/src/controllers/slack/slack-error.utils.d.ts +46 -0
- package/dist/backend/backend/src/controllers/slack/slack-error.utils.d.ts.map +1 -0
- package/dist/backend/backend/src/controllers/slack/slack-error.utils.js +54 -0
- package/dist/backend/backend/src/controllers/slack/slack-error.utils.js.map +1 -0
- package/dist/backend/backend/src/controllers/slack/slack.controller.d.ts.map +1 -1
- package/dist/backend/backend/src/controllers/slack/slack.controller.js +5 -12
- package/dist/backend/backend/src/controllers/slack/slack.controller.js.map +1 -1
- package/dist/backend/backend/src/routes/api.routes.d.ts.map +1 -1
- package/dist/backend/backend/src/routes/api.routes.js +3 -0
- package/dist/backend/backend/src/routes/api.routes.js.map +1 -1
- package/dist/backend/backend/src/services/cloud/mobile-api-relay.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/cloud/mobile-api-relay.service.js +9 -0
- package/dist/backend/backend/src/services/cloud/mobile-api-relay.service.js.map +1 -1
- package/dist/backend/backend/src/services/google/google-api.client.d.ts +23 -2
- package/dist/backend/backend/src/services/google/google-api.client.d.ts.map +1 -1
- package/dist/backend/backend/src/services/google/google-api.client.js +5 -2
- package/dist/backend/backend/src/services/google/google-api.client.js.map +1 -1
- package/dist/backend/backend/src/services/google/google-workspace-token.service.d.ts +61 -11
- package/dist/backend/backend/src/services/google/google-workspace-token.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/google/google-workspace-token.service.js +108 -31
- package/dist/backend/backend/src/services/google/google-workspace-token.service.js.map +1 -1
- package/dist/backend/backend/src/services/slack/slack-team-channel.service.d.ts +24 -0
- package/dist/backend/backend/src/services/slack/slack-team-channel.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/slack/slack-team-channel.service.js +87 -5
- package/dist/backend/backend/src/services/slack/slack-team-channel.service.js.map +1 -1
- package/dist/backend/backend/src/services/slack/slack.service.d.ts +17 -0
- package/dist/backend/backend/src/services/slack/slack.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/slack/slack.service.js +32 -0
- package/dist/backend/backend/src/services/slack/slack.service.js.map +1 -1
- package/dist/backend/backend/src/types/slack.types.d.ts +10 -0
- package/dist/backend/backend/src/types/slack.types.d.ts.map +1 -1
- package/dist/backend/backend/src/types/slack.types.js.map +1 -1
- package/dist/backend/backend/src/utils/incomplete-turn.utils.d.ts +1 -1
- package/dist/backend/backend/src/utils/incomplete-turn.utils.d.ts.map +1 -1
- package/dist/backend/backend/src/utils/incomplete-turn.utils.js +4 -0
- package/dist/backend/backend/src/utils/incomplete-turn.utils.js.map +1 -1
- package/dist/backend/build-info.json +2 -2
- package/dist/cli/backend/src/constants.d.ts +18 -4
- package/dist/cli/backend/src/constants.d.ts.map +1 -1
- package/dist/cli/backend/src/constants.js +16 -4
- package/dist/cli/backend/src/constants.js.map +1 -1
- package/dist/cli/backend/src/services/slack/slack-team-channel.service.d.ts +24 -0
- package/dist/cli/backend/src/services/slack/slack-team-channel.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/slack/slack-team-channel.service.js +87 -5
- package/dist/cli/backend/src/services/slack/slack-team-channel.service.js.map +1 -1
- package/dist/cli/backend/src/services/slack/slack.service.d.ts +17 -0
- package/dist/cli/backend/src/services/slack/slack.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/slack/slack.service.js +32 -0
- package/dist/cli/backend/src/services/slack/slack.service.js.map +1 -1
- package/dist/cli/backend/src/types/slack.types.d.ts +10 -0
- package/dist/cli/backend/src/types/slack.types.d.ts.map +1 -1
- package/dist/cli/backend/src/types/slack.types.js.map +1 -1
- package/dist/cli/backend/src/utils/incomplete-turn.utils.d.ts +1 -1
- package/dist/cli/backend/src/utils/incomplete-turn.utils.d.ts.map +1 -1
- package/dist/cli/backend/src/utils/incomplete-turn.utils.js +4 -0
- package/dist/cli/backend/src/utils/incomplete-turn.utils.js.map +1 -1
- package/frontend/dist/assets/{index-e079a375.js → index-e7785269.js} +267 -267
- package/frontend/dist/index.html +1 -1
- package/package.json +1 -1
- package/packages/crewly-agent/src/eval/desktop/desktop-tasks.test.ts +96 -0
- package/packages/crewly-agent/src/eval/desktop/desktop-tasks.ts +226 -0
- package/packages/crewly-agent/src/runtime/agent-runner.service.test.ts +10 -1
- package/packages/crewly-agent/src/runtime/agent-runner.service.ts +199 -4
- package/packages/crewly-agent/src/runtime/computer.tool.test.ts +219 -0
- package/packages/crewly-agent/src/runtime/computer.tool.ts +405 -0
- package/packages/crewly-agent/src/runtime/desktop-checkpoint.test.ts +136 -0
- package/packages/crewly-agent/src/runtime/desktop-checkpoint.ts +231 -0
- package/packages/crewly-agent/src/runtime/desktop-recovery.test.ts +100 -0
- package/packages/crewly-agent/src/runtime/desktop-recovery.ts +195 -0
- package/packages/crewly-agent/src/runtime/desktop-task-runtime.test.ts +251 -0
- package/packages/crewly-agent/src/runtime/desktop-task-runtime.ts +423 -0
- package/packages/crewly-agent/src/runtime/desktop-task.tool.test.ts +218 -0
- package/packages/crewly-agent/src/runtime/desktop-task.tool.ts +343 -0
- package/packages/crewly-agent/src/runtime/text-tool-calls.test.ts +144 -0
- package/packages/crewly-agent/src/runtime/text-tool-calls.ts +316 -0
- package/packages/crewly-agent/src/runtime/text-tool-salvage.test.ts +190 -0
- package/packages/crewly-agent/src/runtime/tool-registry.test.ts +17 -0
- package/packages/crewly-agent/src/runtime/tool-registry.ts +54 -0
- package/packages/crewly-agent/src/runtime/types.ts +16 -1
- package/config/skills/agent/vnc-browser/SKILL.md +0 -140
|
@@ -0,0 +1,485 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# =============================================================================
|
|
3
|
+
# desktop-guards.sh — safety rails shared by every skill that drives the
|
|
4
|
+
# owner's real mouse, keyboard and screen.
|
|
5
|
+
#
|
|
6
|
+
# Sourced rather than copied because there are already two divergent
|
|
7
|
+
# computer-use implementations in this repo (config/skills/agent/computer-use
|
|
8
|
+
# and .../marketplace/computer-use) with different action sets. Duplicating
|
|
9
|
+
# the rails into both is how they drift apart, and a guard that exists in only
|
|
10
|
+
# one copy is worse than none — it makes the skill look safe.
|
|
11
|
+
#
|
|
12
|
+
# Callers must have sourced _common/lib.sh first (for error_exit) and must set
|
|
13
|
+
# ACTION and INPUT before calling cu_apply_guards.
|
|
14
|
+
#
|
|
15
|
+
# Usage:
|
|
16
|
+
# source "${SCRIPT_DIR}/../../_common/desktop-guards.sh"
|
|
17
|
+
# cu_apply_guards
|
|
18
|
+
# =============================================================================
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
CREWLY_HOME_DIR="${CREWLY_HOME:-$HOME/.crewly}"
|
|
22
|
+
DESKTOP_LOCK="${CREWLY_HOME_DIR}/desktop.lock"
|
|
23
|
+
DESKTOP_LOG="${CREWLY_HOME_DIR}/desktop-actions.jsonl"
|
|
24
|
+
DESKTOP_STOP="${CREWLY_HOME_DIR}/desktop.stop"
|
|
25
|
+
LOCK_TTL_SECONDS="${CREWLY_DESKTOP_LOCK_TTL:-120}"
|
|
26
|
+
HOLDER="${CREWLY_SESSION_NAME:-unknown-agent}"
|
|
27
|
+
|
|
28
|
+
mkdir -p "$CREWLY_HOME_DIR" 2>/dev/null || true
|
|
29
|
+
|
|
30
|
+
# ---------------------------------------------------------------------------
|
|
31
|
+
# cu_fail reason message [extra_json]
|
|
32
|
+
#
|
|
33
|
+
# A refusal the agent can act on. Every guard answers in this one shape so a
|
|
34
|
+
# model does not have to parse prose to tell "you lack permission" from "that
|
|
35
|
+
# element was not found".
|
|
36
|
+
# ---------------------------------------------------------------------------
|
|
37
|
+
cu_fail() {
|
|
38
|
+
local reason="$1" message="$2" extra="${3:-{\}}"
|
|
39
|
+
jq -n --arg a "$ACTION" --arg r "$reason" --arg m "$message" --argjson x "$extra" \
|
|
40
|
+
'{success:false, action:$a, reason:$r, message:$m} + $x'
|
|
41
|
+
exit 1
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
# ---------------------------------------------------------------------------
|
|
45
|
+
# Permission preflight
|
|
46
|
+
#
|
|
47
|
+
# macOS grants TCC permissions to the *process that asks*, which for a skill is
|
|
48
|
+
# whatever launched the agent — Terminal, iTerm, or the crewly backend's node.
|
|
49
|
+
# So the answer names the process, otherwise the owner cannot tell what to tick.
|
|
50
|
+
# ---------------------------------------------------------------------------
|
|
51
|
+
asking_process() {
|
|
52
|
+
# The nearest ancestor the user would recognise in the TCC pane.
|
|
53
|
+
ps -o comm= -p "${PPID:-$$}" 2>/dev/null | sed 's:.*/::' || echo "your terminal"
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
has_screen_recording() {
|
|
57
|
+
# CGPreflightScreenCaptureAccess is a plain C function: importing CoreGraphics
|
|
58
|
+
# is not enough, JXA only sees it once it is bound explicitly.
|
|
59
|
+
osascript -l JavaScript -e '
|
|
60
|
+
ObjC.import("CoreGraphics");
|
|
61
|
+
ObjC.bindFunction("CGPreflightScreenCaptureAccess", ["bool", []]);
|
|
62
|
+
$.CGPreflightScreenCaptureAccess() ? "yes" : "no"
|
|
63
|
+
' 2>/dev/null
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
has_accessibility() {
|
|
67
|
+
osascript -l JavaScript -e 'ObjC.import("ApplicationServices"); $.AXIsProcessTrusted() ? "yes" : "no"' 2>/dev/null
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
require_screen_recording() {
|
|
71
|
+
[ "$(has_screen_recording)" = "yes" ] && return 0
|
|
72
|
+
cu_fail "permission_required" \
|
|
73
|
+
"Screen Recording is not granted to $(asking_process), so every screenshot comes back blank. Grant it, then quit and reopen that app — macOS only applies the change on restart." \
|
|
74
|
+
"$(jq -n --arg p "$(asking_process)" '{permission:"screen-recording", grantTo:$p, howTo:"System Settings → Privacy & Security → Screen & System Audio Recording"}')"
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
require_accessibility() {
|
|
78
|
+
[ "$(has_accessibility)" = "yes" ] && return 0
|
|
79
|
+
cu_fail "permission_required" \
|
|
80
|
+
"Accessibility is not granted to $(asking_process), so clicks and keystrokes are silently discarded. Grant it, then quit and reopen that app." \
|
|
81
|
+
"$(jq -n --arg p "$(asking_process)" '{permission:"accessibility", grantTo:$p, howTo:"System Settings → Privacy & Security → Accessibility"}')"
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
# ---------------------------------------------------------------------------
|
|
85
|
+
# Stop switch
|
|
86
|
+
#
|
|
87
|
+
# A file, so anything can set it: the owner, a hotkey, the backend. Checked
|
|
88
|
+
# before every action rather than only at the start, because the point is to
|
|
89
|
+
# stop an agent that is already mid-task.
|
|
90
|
+
# ---------------------------------------------------------------------------
|
|
91
|
+
require_not_stopped() {
|
|
92
|
+
[ -f "$DESKTOP_STOP" ] || return 0
|
|
93
|
+
cu_fail "stopped_by_user" \
|
|
94
|
+
"Desktop control is stopped. The owner halted it; it stays off until $DESKTOP_STOP is removed." \
|
|
95
|
+
"$(jq -n --arg f "$DESKTOP_STOP" '{stopFile:$f}')"
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
# ---------------------------------------------------------------------------
|
|
99
|
+
# Platform
|
|
100
|
+
#
|
|
101
|
+
# Everything here is macOS: screencapture, AppleScript, CoreGraphics event
|
|
102
|
+
# taps, the Accessibility API. On Linux or Windows the commands are simply
|
|
103
|
+
# absent, and the failure an agent saw was `osascript: command not found` —
|
|
104
|
+
# which reads as a broken install rather than an unsupported platform, so it
|
|
105
|
+
# retried. Saying so plainly is the whole fix until the Linux backend of
|
|
106
|
+
# Phase 6 exists.
|
|
107
|
+
# ---------------------------------------------------------------------------
|
|
108
|
+
require_macos() {
|
|
109
|
+
[ "$(uname -s)" = "Darwin" ] && return 0
|
|
110
|
+
cu_fail "unsupported_platform" \
|
|
111
|
+
"Desktop control is macOS-only for now. On $(uname -s) there is no screen to drive from here — use the browser tools for web work, or a skill for anything with an API." \
|
|
112
|
+
"$(jq -n --arg p "$(uname -s)" '{platform:$p, supported:["Darwin"]}')"
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
# ---------------------------------------------------------------------------
|
|
116
|
+
# Locked screen
|
|
117
|
+
#
|
|
118
|
+
# Accessibility does not fail while the screen is locked — it answers with
|
|
119
|
+
# rubbish. Every window of every app comes back with role AXApplication and no
|
|
120
|
+
# real content, and System Events cannot even name the frontmost process. An
|
|
121
|
+
# agent reading that sees a plausible-looking tree and acts on it, clicking
|
|
122
|
+
# coordinates that belong to nothing (2026-09-20, found while testing the
|
|
123
|
+
# perception layer against a locked Mac).
|
|
124
|
+
#
|
|
125
|
+
# There is also nothing useful to do on a locked screen, so this is a refusal
|
|
126
|
+
# rather than a warning.
|
|
127
|
+
# ---------------------------------------------------------------------------
|
|
128
|
+
screen_is_locked() {
|
|
129
|
+
osascript -l JavaScript -e '
|
|
130
|
+
ObjC.import("CoreGraphics");
|
|
131
|
+
ObjC.bindFunction("CGSessionCopyCurrentDictionary", ["id", []]);
|
|
132
|
+
var d = $.CGSessionCopyCurrentDictionary();
|
|
133
|
+
d && ObjC.unwrap(d.objectForKey("CGSSessionScreenIsLocked")) ? "yes" : "no"
|
|
134
|
+
' 2>/dev/null
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
require_unlocked() {
|
|
138
|
+
[ "$(screen_is_locked)" = "yes" ] || return 0
|
|
139
|
+
cu_fail "screen_locked" \
|
|
140
|
+
"The screen is locked. Accessibility answers with placeholder data while it is, so anything read now would be wrong and anything clicked would land on nothing. Wait for the owner to unlock, or ask them to." \
|
|
141
|
+
'{"recoverable":true}'
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
# ---------------------------------------------------------------------------
|
|
145
|
+
# Mutual exclusion
|
|
146
|
+
#
|
|
147
|
+
# One machine has one mouse and one keyboard focus, so two agents acting at
|
|
148
|
+
# once corrupt each other. The lock carries its holder and an expiry, so a
|
|
149
|
+
# crashed agent cannot wedge the desktop forever.
|
|
150
|
+
# ---------------------------------------------------------------------------
|
|
151
|
+
acquire_desktop_lock() {
|
|
152
|
+
local now holder_existing expires
|
|
153
|
+
now=$(date +%s)
|
|
154
|
+
if [ -f "$DESKTOP_LOCK" ]; then
|
|
155
|
+
holder_existing=$(jq -r '.holder // "unknown"' "$DESKTOP_LOCK" 2>/dev/null || echo unknown)
|
|
156
|
+
expires=$(jq -r '.expiresAt // 0' "$DESKTOP_LOCK" 2>/dev/null || echo 0)
|
|
157
|
+
if [ "$holder_existing" != "$HOLDER" ] && [ "$expires" -gt "$now" ] 2>/dev/null; then
|
|
158
|
+
cu_fail "desktop_busy" \
|
|
159
|
+
"$holder_existing is using the desktop until $(date -r "$expires" '+%H:%M:%S'). Wait, or ask that agent to finish." \
|
|
160
|
+
"$(jq -n --arg h "$holder_existing" --argjson e "$expires" '{heldBy:$h, expiresAt:$e}')"
|
|
161
|
+
fi
|
|
162
|
+
fi
|
|
163
|
+
jq -n --arg h "$HOLDER" --argjson e "$((now + LOCK_TTL_SECONDS))" --arg a "$ACTION" \
|
|
164
|
+
'{holder:$h, expiresAt:$e, lastAction:$a}' > "$DESKTOP_LOCK" 2>/dev/null || true
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
# ---------------------------------------------------------------------------
|
|
168
|
+
# Audit
|
|
169
|
+
#
|
|
170
|
+
# Appended before the action runs, so an action that hangs or crashes the shell
|
|
171
|
+
# still leaves a trace. Screenshots of each step come later (Phase 5).
|
|
172
|
+
# ---------------------------------------------------------------------------
|
|
173
|
+
DESKTOP_SHOTS="${CREWLY_HOME_DIR}/desktop-actions"
|
|
174
|
+
|
|
175
|
+
# Small enough that a day of them costs a few megabytes, large enough to see
|
|
176
|
+
# which window was in front and whether a dialog was open.
|
|
177
|
+
THUMB_WIDTH="${CREWLY_DESKTOP_THUMB_WIDTH:-480}"
|
|
178
|
+
|
|
179
|
+
# capture_thumb <label> → path, or empty when it could not be taken
|
|
180
|
+
#
|
|
181
|
+
# Failure is silent and empty: an action must never fail because its evidence
|
|
182
|
+
# could not be recorded.
|
|
183
|
+
capture_thumb() {
|
|
184
|
+
[ "${CREWLY_DESKTOP_AUDIT_SHOTS:-1}" = "1" ] || return 0
|
|
185
|
+
case "$ACTION" in
|
|
186
|
+
click|move|type|key|scroll|drag|focus|focus-app|open-url|click-text|click-ref|fill-ref) ;;
|
|
187
|
+
# Handing over to a person: the picture of what the agent is stuck on is
|
|
188
|
+
# the most useful thing the owner gets, especially on a phone.
|
|
189
|
+
request-human) ;;
|
|
190
|
+
*) return 0 ;;
|
|
191
|
+
esac
|
|
192
|
+
local day dir file
|
|
193
|
+
day=$(date -u +%Y-%m-%d)
|
|
194
|
+
dir="${DESKTOP_SHOTS}/${day}"
|
|
195
|
+
mkdir -p "$dir" 2>/dev/null || return 0
|
|
196
|
+
file="${dir}/$(date -u +%H%M%S)-$$-$1.jpg"
|
|
197
|
+
screencapture -x -t jpg "$file" 2>/dev/null || return 0
|
|
198
|
+
sips --resampleWidth "$THUMB_WIDTH" "$file" --out "$file" >/dev/null 2>&1 || true
|
|
199
|
+
printf '%s' "$file"
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
log_action() {
|
|
203
|
+
local before
|
|
204
|
+
before=$(capture_thumb before)
|
|
205
|
+
jq -nc --arg t "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg s "$HOLDER" --arg a "$ACTION" \
|
|
206
|
+
--argjson i "$INPUT" --arg b "$before" \
|
|
207
|
+
'{at:$t, session:$s, action:$a, input:$i} + (if $b == "" then {} else {before:$b} end)' \
|
|
208
|
+
>> "$DESKTOP_LOG" 2>/dev/null || true
|
|
209
|
+
# The "after" shot is taken on the way out, once the action has landed, so
|
|
210
|
+
# the pair shows cause and effect rather than two pictures of the same
|
|
211
|
+
# moment. Only for actions that got past every rail — a refusal changed
|
|
212
|
+
# nothing and a second identical picture is just noise.
|
|
213
|
+
CU_LOG_BEFORE="$before"
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
# cu_log_after — called once the action has run.
|
|
217
|
+
cu_log_after() {
|
|
218
|
+
[ -n "${CU_LOG_BEFORE:-}" ] || return 0
|
|
219
|
+
local after
|
|
220
|
+
# Give the UI a moment to actually change; without it the pair is useless.
|
|
221
|
+
sleep 0.35
|
|
222
|
+
after=$(capture_thumb after)
|
|
223
|
+
[ -n "$after" ] || return 0
|
|
224
|
+
jq -nc --arg t "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg s "$HOLDER" --arg a "$ACTION" \
|
|
225
|
+
--arg b "$CU_LOG_BEFORE" --arg f "$after" \
|
|
226
|
+
'{at:$t, session:$s, action:$a, phase:"after", before:$b, after:$f}' \
|
|
227
|
+
>> "$DESKTOP_LOG" 2>/dev/null || true
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
# ---------------------------------------------------------------------------
|
|
231
|
+
# Destructive key combos
|
|
232
|
+
#
|
|
233
|
+
# Blocked outright rather than sent for approval: these are almost never what
|
|
234
|
+
# an automation legitimately wants, and a blocked action the agent can route
|
|
235
|
+
# around beats a prompt the owner learns to click through. An operator who
|
|
236
|
+
# really means it sets CREWLY_DESKTOP_ALLOW_DESTRUCTIVE=1 for that one run.
|
|
237
|
+
# ---------------------------------------------------------------------------
|
|
238
|
+
DESTRUCTIVE_KEYS='^(command|cmd)\+(q|w)$|^(command|cmd)\+(delete|backspace)$|^(command|cmd)\+shift\+(delete|backspace)$|^(command|cmd)\+(option|alt)\+escape$'
|
|
239
|
+
|
|
240
|
+
guard_destructive_key() {
|
|
241
|
+
local key_name="$1" normalized
|
|
242
|
+
normalized=$(printf '%s' "$key_name" | tr '[:upper:]' '[:lower:]' | tr -d ' ')
|
|
243
|
+
printf '%s' "$normalized" | grep -qE "$DESTRUCTIVE_KEYS" || return 0
|
|
244
|
+
[ "${CREWLY_DESKTOP_ALLOW_DESTRUCTIVE:-}" = "1" ] && return 0
|
|
245
|
+
cu_fail "destructive_blocked" \
|
|
246
|
+
"\"$key_name\" quits an app or deletes data, which desktop control refuses by default. Achieve the goal another way, or ask the owner to re-run with CREWLY_DESKTOP_ALLOW_DESTRUCTIVE=1." \
|
|
247
|
+
"$(jq -n --arg k "$key_name" '{key:$k, override:"CREWLY_DESKTOP_ALLOW_DESTRUCTIVE=1"}')"
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
# ---------------------------------------------------------------------------
|
|
251
|
+
# Apps that are never automated
|
|
252
|
+
#
|
|
253
|
+
# Typing into a password manager or the Keychain is indistinguishable from
|
|
254
|
+
# exfiltrating credentials, so focus is refused rather than audited.
|
|
255
|
+
# ---------------------------------------------------------------------------
|
|
256
|
+
DENIED_APPS='^(system settings|system preferences|keychain access|1password|1password 7|1password 8|bitwarden|lastpass|dashlane|passwords)$'
|
|
257
|
+
|
|
258
|
+
guard_denied_app() {
|
|
259
|
+
local app_name="$1" normalized
|
|
260
|
+
normalized=$(printf '%s' "$app_name" | tr '[:upper:]' '[:lower:]')
|
|
261
|
+
printf '%s' "$normalized" | grep -qE "$DENIED_APPS" || return 0
|
|
262
|
+
cu_fail "app_not_allowed" \
|
|
263
|
+
"Desktop control does not drive $app_name — it holds the owner's credentials. Ask the owner to do this part." \
|
|
264
|
+
"$(jq -n --arg a "$app_name" '{app:$a}')"
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
# ---------------------------------------------------------------------------
|
|
268
|
+
# Secure text fields
|
|
269
|
+
#
|
|
270
|
+
# `type` goes to whatever has focus, so without this a password box is just
|
|
271
|
+
# another text box. Fails open when the focused element cannot be read: AX
|
|
272
|
+
# gaps are common and refusing every unreadable field would break normal use.
|
|
273
|
+
# ---------------------------------------------------------------------------
|
|
274
|
+
focused_role() {
|
|
275
|
+
osascript -l JavaScript -e '
|
|
276
|
+
ObjC.import("ApplicationServices");
|
|
277
|
+
function roleOf() {
|
|
278
|
+
var sys = $.AXUIElementCreateSystemWide();
|
|
279
|
+
var appRef = Ref();
|
|
280
|
+
if ($.AXUIElementCopyAttributeValue(sys, $.CFSTR("AXFocusedApplication"), appRef) !== 0) return "";
|
|
281
|
+
var elemRef = Ref();
|
|
282
|
+
if ($.AXUIElementCopyAttributeValue(appRef[0], $.CFSTR("AXFocusedUIElement"), elemRef) !== 0) return "";
|
|
283
|
+
var roleRef = Ref();
|
|
284
|
+
if ($.AXUIElementCopyAttributeValue(elemRef[0], $.CFSTR("AXRole"), roleRef) !== 0) return "";
|
|
285
|
+
return ObjC.unwrap(roleRef[0]) || "";
|
|
286
|
+
}
|
|
287
|
+
roleOf();
|
|
288
|
+
' 2>/dev/null || echo ""
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
guard_secure_input() {
|
|
292
|
+
local role
|
|
293
|
+
role=$(focused_role)
|
|
294
|
+
[ "$role" = "AXSecureTextField" ] || return 0
|
|
295
|
+
cu_fail "secure_field" \
|
|
296
|
+
"The focused field is a password box. Desktop control never types into one — ask the owner to enter it." \
|
|
297
|
+
'{"focusedRole":"AXSecureTextField"}'
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
# ---------------------------------------------------------------------------
|
|
301
|
+
# check-permissions — report both TCC grants without performing any action.
|
|
302
|
+
# ---------------------------------------------------------------------------
|
|
303
|
+
do_check_permissions() {
|
|
304
|
+
local screen ax proc
|
|
305
|
+
screen=$(has_screen_recording); ax=$(has_accessibility); proc=$(asking_process)
|
|
306
|
+
jq -n --arg p "$proc" \
|
|
307
|
+
--argjson s "$([ "$screen" = yes ] && echo true || echo false)" \
|
|
308
|
+
--argjson a "$([ "$ax" = yes ] && echo true || echo false)" \
|
|
309
|
+
'{success:true, action:"check-permissions", askingProcess:$p,
|
|
310
|
+
screenRecording:$s, accessibility:$a,
|
|
311
|
+
ready:($s and $a),
|
|
312
|
+
howTo:"System Settings → Privacy & Security → (Screen & System Audio Recording | Accessibility). Grant to the process named above, then quit and reopen it."}'
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
# ---------------------------------------------------------------------------
|
|
316
|
+
# The perception helper
|
|
317
|
+
#
|
|
318
|
+
# A small Swift binary (AX tree, Vision OCR, display list). Compiled on first
|
|
319
|
+
# use and cached, because shipping a binary in a skill directory means
|
|
320
|
+
# shipping an unsigned one a user cannot verify, and because the source is
|
|
321
|
+
# the thing worth reviewing. Rebuilt whenever the source is newer.
|
|
322
|
+
#
|
|
323
|
+
# Swift rather than JXA: the scripting bridge costs a round trip per
|
|
324
|
+
# attribute, so reading one window took seconds — too slow to do before every
|
|
325
|
+
# action. Raw AXUIElement does it in tens of milliseconds.
|
|
326
|
+
# ---------------------------------------------------------------------------
|
|
327
|
+
PERCEIVE_SRC="${CREWLY_SKILLS_COMMON:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}/desktop-perceive.swift"
|
|
328
|
+
PERCEIVE_BIN="${CREWLY_HOME_DIR}/bin/desktop-perceive"
|
|
329
|
+
|
|
330
|
+
cu_perceive() {
|
|
331
|
+
if [ ! -x "$PERCEIVE_BIN" ] || [ "$PERCEIVE_SRC" -nt "$PERCEIVE_BIN" ]; then
|
|
332
|
+
if ! command -v swiftc >/dev/null 2>&1; then
|
|
333
|
+
cu_fail "toolchain_missing" \
|
|
334
|
+
"Element-level perception needs swiftc, which comes with the Xcode Command Line Tools. Install them with: xcode-select --install" \
|
|
335
|
+
'{"install":"xcode-select --install"}'
|
|
336
|
+
fi
|
|
337
|
+
mkdir -p "$(dirname "$PERCEIVE_BIN")"
|
|
338
|
+
if ! swiftc -O -o "$PERCEIVE_BIN" "$PERCEIVE_SRC" 2>"${CREWLY_HOME_DIR}/desktop-perceive-build.log"; then
|
|
339
|
+
cu_fail "build_failed" \
|
|
340
|
+
"Could not build the perception helper. See ${CREWLY_HOME_DIR}/desktop-perceive-build.log." \
|
|
341
|
+
"$(jq -n --arg l "${CREWLY_HOME_DIR}/desktop-perceive-build.log" '{log:$l}')"
|
|
342
|
+
fi
|
|
343
|
+
fi
|
|
344
|
+
"$PERCEIVE_BIN" "$@"
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
# ---------------------------------------------------------------------------
|
|
348
|
+
# Presence: the banner, the hotkey, and the eye on the owner's own input
|
|
349
|
+
#
|
|
350
|
+
# The browser line has shown a takeover banner since April; the desktop showed
|
|
351
|
+
# nothing, and a pointer that starts moving by itself is the difference
|
|
352
|
+
# between automation and a haunting. The resident process owns no policy — it
|
|
353
|
+
# writes the same desktop.stop / desktop.pause files these rails already read,
|
|
354
|
+
# so it can die without leaving anything un-enforced.
|
|
355
|
+
#
|
|
356
|
+
# Started on demand and refreshed before each action; it exits by itself once
|
|
357
|
+
# the refreshes stop, so a crashed agent cannot leave a banner claiming to be
|
|
358
|
+
# working.
|
|
359
|
+
# ---------------------------------------------------------------------------
|
|
360
|
+
PRESENCE_SRC="${CREWLY_SKILLS_COMMON:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}/desktop-presence.swift"
|
|
361
|
+
PRESENCE_BIN="${CREWLY_HOME_DIR}/bin/desktop-presence"
|
|
362
|
+
DESKTOP_PAUSE="${CREWLY_HOME_DIR}/desktop.pause"
|
|
363
|
+
|
|
364
|
+
cu_presence() {
|
|
365
|
+
# Never fatal: an agent that cannot show a banner should still be stoppable
|
|
366
|
+
# by the rails, and failing the action would be worse than a missing banner.
|
|
367
|
+
[ "${CREWLY_DESKTOP_NO_BANNER:-}" = "1" ] && return 0
|
|
368
|
+
if [ ! -x "$PRESENCE_BIN" ] || [ "$PRESENCE_SRC" -nt "$PRESENCE_BIN" ]; then
|
|
369
|
+
command -v swiftc >/dev/null 2>&1 || return 0
|
|
370
|
+
mkdir -p "$(dirname "$PRESENCE_BIN")"
|
|
371
|
+
swiftc -O -o "$PRESENCE_BIN" "$PRESENCE_SRC" 2>"${CREWLY_HOME_DIR}/desktop-presence-build.log" || return 0
|
|
372
|
+
fi
|
|
373
|
+
"$PRESENCE_BIN" "$@" >/dev/null 2>&1 || true
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
# Show (or refresh) the banner for this action.
|
|
377
|
+
cu_presence_refresh() {
|
|
378
|
+
local goal="${CREWLY_AGENT_GOAL:-$ACTION}"
|
|
379
|
+
cu_presence begin --agent "$HOLDER" --goal "$goal"
|
|
380
|
+
# The window only exists while a foreground process is running; start one
|
|
381
|
+
# if none is.
|
|
382
|
+
pgrep -f "desktop-presence begin .*--foreground" >/dev/null 2>&1 || {
|
|
383
|
+
nohup "$PRESENCE_BIN" begin --agent "$HOLDER" --goal "$goal" --foreground >/dev/null 2>&1 &
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
# ---------------------------------------------------------------------------
|
|
388
|
+
# Paused
|
|
389
|
+
#
|
|
390
|
+
# Distinct from stopped: the owner reached for the mouse, or pressed Pause.
|
|
391
|
+
# The task is not abandoned — it waits, and the same banner resumes it. A
|
|
392
|
+
# refusal rather than a block, because a blocked shell holds the desktop lock
|
|
393
|
+
# and would stop the owner's own agents too.
|
|
394
|
+
# ---------------------------------------------------------------------------
|
|
395
|
+
require_not_paused() {
|
|
396
|
+
[ -f "$DESKTOP_PAUSE" ] || return 0
|
|
397
|
+
cu_fail "paused" \
|
|
398
|
+
"Desktop control is paused — the owner is using the machine. Wait, and try again when they hand it back; the task is not cancelled." \
|
|
399
|
+
'{"recoverable":true}'
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
# ---------------------------------------------------------------------------
|
|
403
|
+
# cu_apply_guards
|
|
404
|
+
#
|
|
405
|
+
# Run every rail that applies to $ACTION. Call once, before dispatch.
|
|
406
|
+
#
|
|
407
|
+
# Action names differ between the two computer-use forks (`focus` vs
|
|
408
|
+
# `focus-app`, and only one has `key`), so the lists below accept both; an
|
|
409
|
+
# action a fork does not have simply never matches.
|
|
410
|
+
# ---------------------------------------------------------------------------
|
|
411
|
+
cu_apply_guards() {
|
|
412
|
+
# Platform first: on the wrong OS not even "check what permissions I have"
|
|
413
|
+
# has an answer, and `osascript: command not found` reads as a broken
|
|
414
|
+
# install rather than an unsupported platform.
|
|
415
|
+
require_macos
|
|
416
|
+
[ "$ACTION" = "check-permissions" ] && return 0
|
|
417
|
+
[ "$ACTION" = "check-accessibility" ] && return 0
|
|
418
|
+
# Listing screens reads no window and touches nothing.
|
|
419
|
+
[ "$ACTION" = "displays" ] && return 0
|
|
420
|
+
# Asking for a person is how an agent gets *out* of being stuck, so it must
|
|
421
|
+
# work while paused — refusing it would leave the agent with nothing to do
|
|
422
|
+
# but retry the thing it already cannot do.
|
|
423
|
+
[ "$ACTION" = "request-human" ] && { require_macos; log_action; return 0; }
|
|
424
|
+
|
|
425
|
+
require_not_stopped
|
|
426
|
+
log_action
|
|
427
|
+
|
|
428
|
+
# Permanent policy first, before anything transient. Both "you may not press
|
|
429
|
+
# ⌘Q" and "the screen is locked" can be true at once, and answering with the
|
|
430
|
+
# transient one invites the agent to wait and retry something that will never
|
|
431
|
+
# be allowed. These are pure string checks, so they also work on a locked
|
|
432
|
+
# screen where reading the UI would not.
|
|
433
|
+
case "$ACTION" in
|
|
434
|
+
key) guard_destructive_key "$(printf '%s' "$INPUT" | jq -r '.key // empty')" ;;
|
|
435
|
+
focus|focus-app|open-url) guard_denied_app "$(printf '%s' "$INPUT" | jq -r '.app // empty')" ;;
|
|
436
|
+
esac
|
|
437
|
+
|
|
438
|
+
# Then the transient conditions, cheapest first.
|
|
439
|
+
require_not_paused
|
|
440
|
+
require_unlocked
|
|
441
|
+
|
|
442
|
+
case "$ACTION" in
|
|
443
|
+
screenshot|find|click-text|ocr) require_screen_recording ;;
|
|
444
|
+
esac
|
|
445
|
+
case "$ACTION" in
|
|
446
|
+
click|move|type|key|scroll|drag|focus|focus-app|open-url|list-apps|click-text|read-ui|get-text|\
|
|
447
|
+
snapshot|click-ref|fill-ref|resolve|wait-for)
|
|
448
|
+
require_accessibility ;;
|
|
449
|
+
esac
|
|
450
|
+
|
|
451
|
+
# Reads the focused element, so it has to come after the lock and permission
|
|
452
|
+
# checks — on a locked screen the answer would be meaningless.
|
|
453
|
+
# fill-ref's own secure-field refusal lives in the perception helper, which
|
|
454
|
+
# reads the target element's role directly instead of guessing from focus.
|
|
455
|
+
case "$ACTION" in
|
|
456
|
+
type) guard_secure_input ;;
|
|
457
|
+
esac
|
|
458
|
+
|
|
459
|
+
# Only actions that change something take the lock. Looking (snapshot, ocr,
|
|
460
|
+
# resolve, wait-for) must stay free: an agent waiting its turn still needs
|
|
461
|
+
# to see, and two agents reading at once cannot corrupt anything.
|
|
462
|
+
case "$ACTION" in
|
|
463
|
+
click|move|type|key|scroll|drag|focus|focus-app|open-url|click-text|click-ref|fill-ref)
|
|
464
|
+
acquire_desktop_lock ;;
|
|
465
|
+
esac
|
|
466
|
+
|
|
467
|
+
# Dry run: every rail above has passed, so the action *would* be allowed —
|
|
468
|
+
# report that and stop before touching the mouse. Exists because the rails
|
|
469
|
+
# can only be tested honestly by also testing what they let through, and a
|
|
470
|
+
# test suite must not open apps or send keystrokes on the owner's machine.
|
|
471
|
+
# Agents can use it the same way, to check an action before committing to it.
|
|
472
|
+
if [ "${CREWLY_DESKTOP_DRY_RUN:-}" = "1" ]; then
|
|
473
|
+
jq -n --arg a "$ACTION" '{success:true, action:$a, dryRun:true, wouldRun:true}'
|
|
474
|
+
exit 0
|
|
475
|
+
fi
|
|
476
|
+
|
|
477
|
+
# The banner goes up last, and only for actions that will actually move
|
|
478
|
+
# something. Reading the screen is not taking the machine over, and a dry
|
|
479
|
+
# run moves nothing at all — raising it for either would cry wolf, and a
|
|
480
|
+
# banner the owner learns to ignore is worse than none.
|
|
481
|
+
case "$ACTION" in
|
|
482
|
+
click|move|type|key|scroll|drag|focus|focus-app|open-url|click-text|click-ref|fill-ref)
|
|
483
|
+
cu_presence_refresh ;;
|
|
484
|
+
esac
|
|
485
|
+
}
|