crewly 1.20.40 → 1.20.48

package/config/skills/agent/computer-use/execute.sh

@@ -35,6 +35,14 @@ INPUT=$(read_json_input "${1:-}")
 ACTION=$(echo "$INPUT" | jq -r '.action // empty')
 [ -z "$ACTION" ] && error_exit "Missing required parameter: action"
 
+# ---------------------------------------------------------------------------
+# Safety rails (permissions, stop switch, desktop lock, destructive-key and
+# secure-field refusals, audit) — shared with the marketplace fork.
+# ---------------------------------------------------------------------------
+source "${SCRIPT_DIR}/../../_common/desktop-guards.sh"
+cu_apply_guards
+
+
 # ---------------------------------------------------------------------------
 # Helper: get screen dimensions and scale factor
 # ---------------------------------------------------------------------------
@@ -56,9 +64,15 @@ get_screen_info() {
 # optionally overlays a grid, optionally crops.
 # ---------------------------------------------------------------------------
 do_screenshot() {
-  local output="${TMPDIR_CU}/screen_$(date +%s%N).png"
+  local output=$(echo "$INPUT" | jq -r '.output // empty')
+  [ -z "$output" ] && output="${TMPDIR_CU}/screen_$(date +%s%N).png"
   local grid=$(echo "$INPUT" | jq -r '.grid // empty')
   local crop_json=$(echo "$INPUT" | jq -r '.crop // empty')
+  # Cap the width the caller wants to reason in. A model's pointing accuracy
+  # falls off on a wide image because it is downsampled before the model sees
+  # it, and one reasoning in the original coordinate space then points at the
+  # wrong place — so the caller asks for the width it will think in.
+  local max_width=$(echo "$INPUT" | jq -r '.maxWidth // empty')
 
   # Capture full screen (silent)
   screencapture -x "$output"
@@ -82,6 +96,15 @@ do_screenshot() {
   local img_h
   img_h=$(sips -g pixelHeight "$output" | tail -1 | awk '{print $2}')
 
+  # Then, if asked, shrink further to the caller's reasoning width. Never
+  # enlarge: a small screen is already easier to point at, and scaling up
+  # would invent precision the model does not have.
+  if [ -n "$max_width" ] && [ "$max_width" -gt 0 ] 2>/dev/null && [ "$img_w" -gt "$max_width" ]; then
+    sips --resampleWidth "$max_width" "$output" --out "$output" >/dev/null 2>&1
+    img_w=$(sips -g pixelWidth "$output" | tail -1 | awk '{print $2}')
+    img_h=$(sips -g pixelHeight "$output" | tail -1 | awk '{print $2}')
+  fi
+
   # Crop FIRST, then grid — so grid labels show absolute screen coordinates
   local origin_x=0 origin_y=0
   if [ -n "$crop_json" ]; then
@@ -289,7 +312,9 @@ do_key() {
   if [ ${#parts[@]} -eq 1 ]; then
     key_name="${parts[0]}"
   else
-    key_name="${parts[-1]}"
+    # macOS ships bash 3.2, which has no negative array indices — using one
+    # here made every modifier combo fail with "bad array subscript".
+    key_name="${parts[$(( ${#parts[@]} - 1 ))]}"
     for ((i=0; i<${#parts[@]}-1; i++)); do
       case "${parts[$i]}" in
         command|cmd)   modifiers="${modifiers}command down, " ;;
@@ -844,9 +869,221 @@ do_click_at() {
   " >/dev/null 2>&1
 }
 
+# =============================================================================
+# Element-level perception and action (Phase 2)
+#
+# Everything above works in screen coordinates: the agent takes a screenshot,
+# guesses where a button is, and clicks a point. That misses, costs an image
+# per step, and breaks on a different theme or window position.
+#
+# These work in elements. `snapshot` returns refs (@e1, @e2 …) with roles,
+# names and frames; `click`/`type` accept a ref and go through the
+# accessibility action, which does not care where the window sits or whether
+# something overlaps it.
+# =============================================================================
+
+# ---------------------------------------------------------------------------
+# snapshot — the elements of an app, as refs the agent can act on.
+# ---------------------------------------------------------------------------
+do_snapshot() {
+  local app max flags=()
+  app=$(printf '%s' "$INPUT" | jq -r '.app // empty')
+  max=$(printf '%s' "$INPUT" | jq -r '.max // empty')
+  [ -n "$app" ] && flags+=(--app "$app")
+  [ -n "$max" ] && flags+=(--max "$max")
+  [ "$(printf '%s' "$INPUT" | jq -r '.allWindows // false')" = "true" ] && flags+=(--all-windows)
+  [ "$(printf '%s' "$INPUT" | jq -r '.menus // false')" = "true" ] && flags+=(--menus)
+  cu_perceive snapshot ${flags[@]+"${flags[@]}"}
+}
+
+# ---------------------------------------------------------------------------
+# click-ref — press an element by ref.
+#
+# AXPress first: it reaches a button that is scrolled out of view or covered
+# by another window, which a coordinate click cannot. Falls back to clicking
+# the element's centre for the many controls that expose no press action.
+# ---------------------------------------------------------------------------
+do_click_ref() {
+  local ref result ok cx cy
+  ref=$(printf '%s' "$INPUT" | jq -r '.ref // empty')
+  require_param "ref" "$ref"
+
+  result=$(cu_perceive press --ref "$ref" 2>&1) || { printf '%s\n' "$result"; exit 1; }
+  ok=$(printf '%s' "$result" | jq -r '.success // false')
+  if [ "$ok" = "true" ]; then
+    printf '%s' "$result" | jq -c '{success:true, action:"click-ref", ref:.ref, method:"AXPress"}'
+    return 0
+  fi
+
+  # No press action (or it refused) — click where the element is.
+  cx=$(printf '%s' "$result" | jq -r '.center[0] // empty')
+  cy=$(printf '%s' "$result" | jq -r '.center[1] // empty')
+  if [ -z "$cx" ] || [ -z "$cy" ]; then
+    printf '%s\n' "$result"
+    exit 1
+  fi
+  do_click_at "$cx" "$cy"
+  jq -n --arg r "$ref" --argjson x "$cx" --argjson y "$cy" \
+    '{success:true, action:"click-ref", ref:$r, method:"coordinate-fallback", x:$x, y:$y}'
+}
+
+# ---------------------------------------------------------------------------
+# fill-ref — set a field's value directly.
+#
+# More reliable than typing: no dependence on focus, on the keyboard layout,
+# or on an input method being in the right mode.
+# ---------------------------------------------------------------------------
+do_fill_ref() {
+  local ref text
+  ref=$(printf '%s' "$INPUT" | jq -r '.ref // empty')
+  text=$(printf '%s' "$INPUT" | jq -r '.text // empty')
+  require_param "ref" "$ref"
+  require_param "text" "$text"
+  cu_perceive set-value --ref "$ref" --text "$text"
+}
+
+# ---------------------------------------------------------------------------
+# resolve — what is at this ref now.
+#
+# Worth calling after something may have changed the window: it reports
+# whether the element still matches what the snapshot recorded.
+# ---------------------------------------------------------------------------
+do_resolve() {
+  local ref
+  ref=$(printf '%s' "$INPUT" | jq -r '.ref // empty')
+  require_param "ref" "$ref"
+  cu_perceive resolve --ref "$ref"
+}
+
+# ---------------------------------------------------------------------------
+# ocr — the text on screen and where it is.
+#
+# Covers what accessibility does not expose: a canvas, a PDF page, an app
+# that simply does not implement AX. Local and free (macOS Vision).
+# ---------------------------------------------------------------------------
+do_ocr() {
+  local region flags=()
+  region=$(printf '%s' "$INPUT" | jq -r 'if .region then "\(.region.x),\(.region.y),\(.region.w),\(.region.h)" else empty end')
+  [ -n "$region" ] && flags+=(--region "$region")
+  local image; image=$(printf '%s' "$INPUT" | jq -r '.image // empty')
+  [ -n "$image" ] && flags+=(--image "$image")
+  cu_perceive ocr ${flags[@]+"${flags[@]}"}
+}
+
+# ---------------------------------------------------------------------------
+# displays — every screen and its frame, so coordinates are unambiguous.
+# ---------------------------------------------------------------------------
+do_displays() {
+  cu_perceive displays
+}
+
+# ---------------------------------------------------------------------------
+# wait-for — block until the screen is in the expected state.
+#
+# The alternative is sleeping and hoping: an agent that clicks during an
+# animation hits nothing, and one that sleeps long enough to be safe is slow
+# on every step instead.
+#
+#   {"action":"wait-for","app":"Numbers"}        that app is frontmost
+#   {"action":"wait-for","ref":"@e12"}           that element resolves
+#   {"action":"wait-for","text":"Export"}        that text is on screen
+#   {"action":"wait-for","idle":true}            the screen stopped changing
+# ---------------------------------------------------------------------------
+do_wait_for() {
+  local app ref text idle timeout deadline
+  app=$(printf '%s' "$INPUT" | jq -r '.app // empty')
+  ref=$(printf '%s' "$INPUT" | jq -r '.ref // empty')
+  text=$(printf '%s' "$INPUT" | jq -r '.text // empty')
+  idle=$(printf '%s' "$INPUT" | jq -r '.idle // false')
+  timeout=$(printf '%s' "$INPUT" | jq -r '.timeoutMs // 10000')
+  deadline=$(( $(date +%s) + timeout / 1000 ))
+
+  if [ -z "$app" ] && [ -z "$ref" ] && [ -z "$text" ] && [ "$idle" != "true" ]; then
+    error_exit "wait-for needs one of: app, ref, text, idle"
+  fi
+
+  local previous="" current
+  while [ "$(date +%s)" -le "$deadline" ]; do
+    if [ -n "$app" ]; then
+      local front
+      front=$(osascript -e 'tell application "System Events" to get name of first process whose frontmost is true' 2>/dev/null)
+      if [ "$front" = "$app" ]; then
+        jq -n --arg a "$app" '{success:true, action:"wait-for", matched:"app", app:$a}'; return 0
+      fi
+    fi
+    if [ -n "$ref" ]; then
+      if cu_perceive resolve --ref "$ref" 2>/dev/null | jq -e '.success and .matches' >/dev/null 2>&1; then
+        jq -n --arg r "$ref" '{success:true, action:"wait-for", matched:"ref", ref:$r}'; return 0
+      fi
+    fi
+    if [ -n "$text" ]; then
+      if cu_perceive ocr 2>/dev/null | jq -e --arg t "$text" '[.items[]?.text | select(contains($t))] | length > 0' >/dev/null 2>&1; then
+        jq -n --arg t "$text" '{success:true, action:"wait-for", matched:"text", text:$t}'; return 0
+      fi
+    fi
+    if [ "$idle" = "true" ]; then
+      # Two consecutive identical frames mean nothing is animating.
+      local shot="${TMPDIR_CU}/idle-$$.png"
+      screencapture -x -t jpg "$shot" 2>/dev/null
+      current=$(shasum -a 1 "$shot" 2>/dev/null | cut -d' ' -f1)
+      rm -f "$shot"
+      if [ -n "$previous" ] && [ "$current" = "$previous" ]; then
+        jq -n '{success:true, action:"wait-for", matched:"idle"}'; return 0
+      fi
+      previous="$current"
+    fi
+    sleep 0.4
+  done
+
+  cu_fail "wait_timeout" \
+    "Nothing matched within ${timeout}ms. Take a snapshot to see what is actually on screen." \
+    "$(jq -n --argjson t "$timeout" '{timeoutMs:$t}')"
+}
+
+# ---------------------------------------------------------------------------
+# request-human — hand the machine back for something an agent must not do.
+#
+# The deleted `vnc-browser` skill described this and shipped no script. The
+# honest version is smaller than a VNC tunnel and covers the cases that
+# actually arise: a CAPTCHA, a two-factor prompt, a login, a decision the
+# owner has to make. The agent pauses itself, says what it needs and why, and
+# waits — rather than trying to be clever, which for a login means typing a
+# password it must never type.
+#
+# Pausing (not stopping) is the point: the owner deals with it and the task
+# picks up, using the same banner button they already have.
+# ---------------------------------------------------------------------------
+do_request_human() {
+  local reason detail shot
+  reason=$(printf '%s' "$INPUT" | jq -r '.reason // empty')
+  detail=$(printf '%s' "$INPUT" | jq -r '.detail // empty')
+  require_param "reason" "$reason"
+
+  # A picture of what the agent is stuck on is worth more than its
+  # description of it, and the owner may be reading this on a phone.
+  shot=$(capture_thumb handover 2>/dev/null || true)
+
+  # Pause rather than stop: the task is not cancelled, it is waiting.
+  printf 'awaiting-human\n' > "$DESKTOP_PAUSE" 2>/dev/null || true
+  cu_presence begin --agent "$HOLDER" --goal "needs you: $reason"
+
+  jq -n --arg r "$reason" --arg d "$detail" --arg s "$HOLDER" --arg shot "$shot" \
+    '{success:true, action:"request-human", waiting:true, reason:$r,
+      detail:$d, agent:$s,
+      note:"Desktop control is paused until the owner resumes it (banner → Resume, or remove ~/.crewly/desktop.pause). Do not try to work around this."}
+     + (if $shot == "" then {} else {screenshot:$shot} end)'
+}
+
 # ---------------------------------------------------------------------------
 # Action dispatch
+#
+# The "after" audit shot is taken on the way out, so the log holds a pair
+# showing what the action actually changed — the point of keeping evidence at
+# all. It runs on every exit path, including a failure, because a failed
+# action that moved something is exactly what one wants to look at later.
 # ---------------------------------------------------------------------------
+trap 'cu_log_after 2>/dev/null || true' EXIT
+
 case "$ACTION" in
   screenshot)  do_screenshot ;;
   click)       do_click ;;
@@ -860,5 +1097,14 @@ case "$ACTION" in
   list-apps)   do_list_apps ;;
   find)        do_find ;;
   click-text)  do_click_text ;;
-  *)           error_exit "Unknown action: $ACTION. Valid: screenshot, click, move, type, key, scroll, drag, focus, open-url, list-apps, find, click-text" ;;
+  check-permissions) do_check_permissions ;;
+  snapshot)    do_snapshot ;;
+  click-ref)   do_click_ref ;;
+  fill-ref)    do_fill_ref ;;
+  resolve)     do_resolve ;;
+  ocr)         do_ocr ;;
+  displays)    do_displays ;;
+  wait-for)    do_wait_for ;;
+  request-human) do_request_human ;;
+  *)           error_exit "Unknown action: $ACTION. Valid: screenshot, click, move, type, key, scroll, drag, focus, open-url, list-apps, find, click-text, check-permissions, snapshot, click-ref, fill-ref, resolve, ocr, displays, wait-for, request-human" ;;
 esac

package/config/skills/agent/desktop-app-control/SKILL.md

@@ -142,3 +142,22 @@ bash execute.sh snapshot --session vscode --interactive
 - `agent-browser` (npm install -g agent-browser && agent-browser install)
 - macOS / Linux / Windows
 - Target apps must be relaunched with `--remote-debugging-port` flag
+
+## Which control surface to use
+
+Crewly has three ways to act on a screen. Pick the **lowest** one that can do
+the job — each step down costs more tokens, breaks more easily, and disturbs
+the user more.
+
+| Need | Use | Why |
+|---|---|---|
+| Anything a Crewly skill or connector already does (mail, Drive, Slack, calendar, git, files) | that skill | No screen at all. Fastest and cannot misclick. |
+| Content of a web page, or acting as the signed-in user in Chrome | `remote-browser` | Real Chrome, real session, per-agent bound tab, and the user sees a takeover banner. |
+| An Electron app (VS Code, Slack, Notion, Figma) | `desktop-app-control` | Accessibility snapshot with element refs — no coordinates. Needs the app started with a debug port. |
+| Native macOS apps, system dialogs, anything the above cannot reach | `computer-use` | Last resort: screen coordinates and pixels. Slowest and most fragile. |
+
+`computer-use` refuses destructive key combos, typing into password fields and
+driving credential apps, holds a machine-wide lock while it works, and logs
+every action to `~/.crewly/desktop-actions.jsonl`. Run
+`{"action":"check-permissions"}` first — without Screen Recording and
+Accessibility every action fails, and the refusal tells you what to grant.

package/config/skills/agent/remote-browser/SKILL.md

@@ -176,3 +176,22 @@ bash execute.sh -a unbind-tab
   they block the extension until dismissed by hand.
 - Always `unbind-tab` when a job is done, or abandoned tabs accumulate in the
   user's window.
+
+## Which control surface to use
+
+Crewly has three ways to act on a screen. Pick the **lowest** one that can do
+the job — each step down costs more tokens, breaks more easily, and disturbs
+the user more.
+
+| Need | Use | Why |
+|---|---|---|
+| Anything a Crewly skill or connector already does (mail, Drive, Slack, calendar, git, files) | that skill | No screen at all. Fastest and cannot misclick. |
+| Content of a web page, or acting as the signed-in user in Chrome | `remote-browser` | Real Chrome, real session, per-agent bound tab, and the user sees a takeover banner. |
+| An Electron app (VS Code, Slack, Notion, Figma) | `desktop-app-control` | Accessibility snapshot with element refs — no coordinates. Needs the app started with a debug port. |
+| Native macOS apps, system dialogs, anything the above cannot reach | `computer-use` | Last resort: screen coordinates and pixels. Slowest and most fragile. |
+
+`computer-use` refuses destructive key combos, typing into password fields and
+driving credential apps, holds a machine-wide lock while it works, and logs
+every action to `~/.crewly/desktop-actions.jsonl`. Run
+`{"action":"check-permissions"}` first — without Screen Recording and
+Accessibility every action fails, and the refusal tells you what to grant.

package/dist/backend/backend/src/controllers/desktop/desktop.controller.d.ts

@@ -0,0 +1,105 @@
+/**
+ * Desktop control over HTTP.
+ *
+ * Phase 6 of docs/research/computer-use-capability-assessment.md. Until now
+ * desktop control was reachable only by a process on the machine itself: an
+ * agent in a shell, running the skill. Nothing at the portal, on the phone,
+ * or on another Mac could see or touch it, so "check what that machine is
+ * doing" meant walking over to it.
+ *
+ * This is the same shape `/api/browser/*` already has, deliberately. It does
+ * not invent a transport: the REST-over-relay path the mobile app uses
+ * carries these routes once they are allowlisted, so the portal and the phone
+ * reach a machine's desktop through machinery that already exists and is
+ * already authenticated.
+ *
+ * Every route shells out to the computer-use skill. That keeps the safety
+ * rails — permissions, stop switch, pause, desktop lock, destructive-key and
+ * password-field refusals, the banner, the audit log — in one place. A second
+ * implementation behind an HTTP route would be a second thing to keep in
+ * step, and it would be the one reachable from the internet.
+ *
+ * @module controllers/desktop/desktop.controller
+ */
+import type { Request, Response } from 'express';
+/**
+ * Actions this route will run.
+ *
+ * An allowlist rather than a pass-through: these routes are reachable from
+ * the relay, so the set of things the internet can ask a Mac to do should be
+ * written down in one visible place rather than inferred from whatever the
+ * skill happens to support today.
+ *
+ * `read` actions are safe to expose broadly. `act` actions move the mouse and
+ * keyboard, so they are gated separately below.
+ */
+export declare const DESKTOP_READ_ACTIONS: Set<string>;
+export declare const DESKTOP_ACT_ACTIONS: Set<string>;
+/**
+ * Run the skill and return whatever JSON it printed.
+ *
+ * The skill's own refusals are passed through untouched: they already name
+ * the reason and what to do about it, and rewording them at this layer would
+ * blur the one thing a caller needs.
+ *
+ * @param input - Skill payload
+ * @param agentSession - Who is asking, for the audit log and the banner
+ * @returns The parsed reply and the HTTP status to send it with
+ */
+export declare function runDesktopAction(input: Record<string, unknown>, agentSession?: string): Promise<{
+    status: number;
+    body: Record<string, unknown>;
+}>;
+/**
+ * Parse the skill's output, which is one JSON object possibly preceded by a
+ * shell warning.
+ *
+ * @param raw - Captured stdout/stderr
+ * @returns The object, or a described failure
+ */
+export declare function parseSkillJson(raw: string): Record<string, unknown>;
+/**
+ * Decide whether an action may run, and say why not.
+ *
+ * @param action - Requested action
+ * @param allowActing - Whether mouse/keyboard actions are permitted here
+ * @returns null when allowed, otherwise the refusal body
+ */
+export declare function checkAction(action: string, allowActing: boolean): Record<string, unknown> | null;
+/**
+ * POST /api/desktop/act — run one desktop action.
+ *
+ * @param req - Body is the skill payload, `{ action, … }`
+ * @param res - The skill's own JSON
+ */
+export declare function desktopAct(req: Request, res: Response): Promise<void>;
+/**
+ * POST /api/desktop/look — the reading half, for callers that may watch but
+ * not touch (the portal showing what a machine is doing, say).
+ *
+ * @param req - Body is the skill payload
+ * @param res - The skill's own JSON
+ */
+export declare function desktopLook(req: Request, res: Response): Promise<void>;
+/**
+ * GET /api/desktop/status — is desktop control usable, and is anyone using it.
+ *
+ * The one call the portal needs to show a machine's state, and the one that
+ * must work even when everything else is refused.
+ *
+ * @param _req - Unused
+ * @param res - Permissions, presence, and whether it is paused or stopped
+ */
+export declare function desktopStatus(_req: Request, res: Response): Promise<void>;
+/**
+ * POST /api/desktop/stop — halt everything, from anywhere.
+ *
+ * The remote twin of the ⌃⌥⌘. hotkey, so an owner who is not at the machine
+ * can still take the mouse back. Deliberately not gated behind the acting
+ * allowlist: stopping is always allowed.
+ *
+ * @param req - Body `{ resume: true }` to lift it instead
+ * @param res - The new state
+ */
+export declare function desktopStop(req: Request, res: Response): Promise<void>;
+//# sourceMappingURL=desktop.controller.d.ts.map

package/dist/backend/backend/src/controllers/desktop/desktop.controller.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"desktop.controller.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/controllers/desktop/desktop.controller.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAIH,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAYjD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,oBAAoB,aAE/B,CAAC;AAEH,eAAO,MAAM,mBAAmB,aAE9B,CAAC;AAQH;;;;;;;;;;GAUG;AACH,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC9B,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,CAAC,CAgC5D;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CA0BnE;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAoBhG;AAUD;;;;;GAKG;AACH,wBAAsB,UAAU,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAY3E;AAED;;;;;;GAMG;AACH,wBAAsB,WAAW,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAU5E;AAED;;;;;;;;GAQG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAU/E;AAED;;;;;;;;;GASG;AACH,wBAAsB,WAAW,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAiB5E"}