crewly 1.20.35 → 1.20.47

package/packages/crewly-agent/src/runtime/tool-registry.ts

@@ -19,6 +19,8 @@ import { KEY_EXTRACTION_BLOCKED_COMMANDS, PromptGuardService } from './prompt-gu
 import { EnvIsolationService } from './env-isolation.service.js';
 import { OutputFilterService } from './output-filter.service.js';
 import { createWebSearchTool } from './web-search.tool.js';
+import { createComputerTool } from './computer.tool.js';
+import { createDesktopTaskTool } from './desktop-task.tool.js';
 
 /** TTL for delegation idle event subscriptions (minutes) */
 const DELEGATION_SUBSCRIPTION_TTL_MINUTES = 120;
@@ -48,6 +50,15 @@ const BLOCKED_COMMAND_PATTERNS: RegExp[] = [
   /\bdd\s+.*of=\/dev\//i,                    // dd of=/dev/...
   /\blaunchctl\b/i,
   /\bsystemctl\b/i,
+  // Desktop-control guards, made untamperable from inside an agent's own
+  // command. The computer-use skill refuses destructive key combos unless
+  // CREWLY_DESKTOP_ALLOW_DESTRUCTIVE is set, and stops entirely while
+  // ~/.crewly/desktop.stop exists — both of which an agent could otherwise
+  // simply set or delete in the same shell line, making the guard theatre.
+  // The owner can still set the variable in the agent's environment, which
+  // is the intended escape hatch (2026-09-20).
+  /\bCREWLY_DESKTOP_ALLOW_DESTRUCTIVE\b/i,
+  /\.crewly\/desktop\.(stop|lock)\b/i,
   // Key extraction protection (from prompt-guard.service.ts)
   ...KEY_EXTRACTION_BLOCKED_COMMANDS,
 ];
@@ -557,6 +568,11 @@ export const TOOL_SENSITIVITY: Record<string, ToolSensitivity> = {
   handoff_task: 'sensitive',
   // Cloud-backed web search
   web_search: 'safe',
+  // Desktop control: it moves the owner's real mouse and keyboard.
+  computer: 'destructive',
+  // The task layer only plans and verifies; the acting happens through
+  // `computer`, which carries the destructive rating.
+  desktop_task: 'sensitive',
 };
 
 /**
@@ -700,6 +716,21 @@ function sanitizeArgs(args: Record<string, unknown>): Record<string, unknown> {
  * @param callbacks - Optional callbacks for compaction and audit logging
  * @returns Object of named tools ready to pass to generateText
  */
+/**
+ * Translate a skill-shaped payload into `computer` tool arguments.
+ *
+ * The task layer asks in the skill's vocabulary (`snapshot`, `ocr`); the tool
+ * speaks the Anthropic-aligned one. One small map here beats teaching the
+ * task layer a second vocabulary it has no other use for.
+ *
+ * @param input - Skill payload
+ * @returns Arguments for the computer tool
+ */
+function desktopActionToToolArgs(input: Record<string, unknown>): Record<string, unknown> {
+  const action = String(input['action'] ?? '');
+  return { action, ...(input['app'] ? { app: input['app'] } : {}) };
+}
+
 export function createTools(client: CrewlyApiClient, sessionName: string, projectPath?: string, callbacks?: ToolCallbacks, conversationId?: string, slackContext?: { channelId: string; threadTs?: string }, mcpTools?: Record<string, ToolDefinition>): Record<string, ToolDefinition> {
   // Slack rate-limiting state: throttle messages within a 3-second window
   let lastSlackSendMs = 0;
@@ -2036,6 +2067,29 @@ export function createTools(client: CrewlyApiClient, sessionName: string, projec
     // ===== Cloud-Backed Web Search =====
 
     web_search: createWebSearchTool(),
+    // Desktop control. Every action goes through the computer-use skill, so
+    // the safety rails live in one place for every runtime rather than being
+    // reimplemented — and kept in step — here.
+    computer: createComputerTool(),
+    // The layer above it: subgoals whose completion is decided by a
+    // checkpoint rather than by the agent's own account of what happened.
+    // It plans and verifies; the acting still goes through `computer`, and
+    // so through the one set of safety rails.
+    desktop_task: createDesktopTaskTool(sessionName, {
+      perceive: async (input) => {
+        const computer = createComputerTool();
+        const result = await computer.execute(desktopActionToToolArgs(input));
+        return (result ?? {}) as Record<string, unknown>;
+      },
+      ...(callbacks?.onEnqueueApproval
+        ? {
+            approvals: {
+              enqueue: (_session: string, tool: string, sensitivity: ToolSensitivity, args: Record<string, unknown>) =>
+                ({ id: callbacks.onEnqueueApproval!(tool, sensitivity, args).approvalId }) as never,
+            },
+          }
+        : {}),
+    }),
   };
 
   // Apply sensitivity classifications and audit wrapping

package/packages/crewly-agent/src/runtime/types.ts

@@ -220,7 +220,16 @@ export type IncompleteReason =
   /** Hit `maxSteps` with work still outstanding. */
   | 'steps-exhausted'
   /** Provider refused on content grounds. */
-  | 'content-filter';
+  | 'content-filter'
+  /**
+   * A desktop task ended with subgoals whose checkpoints never held.
+   *
+   * The turn itself may have finished perfectly well — the model stopped when
+   * it meant to. That is precisely the case worth catching: an agent that
+   * believes it saved the file and says so is indistinguishable, from the
+   * outside, from one that did.
+   */
+  | 'desktop-unverified';
 
 /** Describes a turn that ended early. */
 export interface IncompleteRun {
@@ -556,6 +565,12 @@ export const CREWLY_AGENT_DEFAULTS = {
   MAX_CONTINUATIONS: 3,
   /** Re-runs allowed after the provider ended a turn abnormally. */
   MAX_ABNORMAL_RETRIES: 1,
+  /** Rounds allowed for executing tool calls a model wrote as text instead of calling. */
+  MAX_TEXT_TOOL_SALVAGES: 3,
+  /** Tool calls executed per salvage round, so one confused turn cannot fan out. */
+  MAX_SALVAGED_CALLS_PER_ROUND: 5,
+  /** Characters of a salvaged tool result fed back to the model. */
+  SALVAGED_RESULT_MAX_CHARS: 4000,
   /** Maximum tool calls allowed per single response to prevent polling dead-loops */
   MAX_TOOL_CALLS_PER_RESPONSE: 15,
   /** Consecutive identical tool calls before aborting (loop detection) */

package/config/skills/agent/vnc-browser/SKILL.md

@@ -1,140 +0,0 @@
----
-name: VNC Remote Browser Access
-description: Launch a virtual display with VNC + noVNC + cloudflared for remote browser viewing and control. Enables human-in-the-loop login/verification when Playwright needs manual interaction.
-version: 1.0.0
-category: browser
-skillType: claude-skill
-assignableRoles:
-  - developer
-  - qa
-  - qa-engineer
-  - fullstack-dev
-  - backend-developer
-  - frontend-developer
-  - generalist
-triggers:
-  - vnc
-  - remote browser
-  - browser login
-  - manual login
-  - human verification
-  - captcha
-  - 2fa
-  - playwright headful
-tags:
-  - vnc
-  - browser
-  - remote-access
-  - playwright
-  - noVNC
-  - cloudflared
-execution:
-  type: script
-  script:
-    file: execute.sh
-    interpreter: bash
-    timeoutMs: 60000
----
-
-# VNC Remote Browser Access
-
-Bridge macOS Screen Sharing to a public HTTPS URL via noVNC + cloudflared. This lets humans remotely view and control the Mac desktop (and any browser on it) from anywhere — essential when Playwright hits a login page, CAPTCHA, or 2FA prompt.
-
-The stack: **macOS Screen Sharing** (built-in VNC, port 5900) → **websockify + noVNC** (web client, port 6080) → **cloudflared** (public HTTPS tunnel).
-
-## Prerequisites
-
-**macOS Screen Sharing must be enabled manually** before using this skill:
-
-1. Open **System Settings** → **General** → **Sharing**
-2. Turn on **Screen Sharing**
-3. (Optional) Set a VNC password under Screen Sharing options
-
-This only needs to be done once. Screen Sharing runs as a system service and persists across reboots.
-
-## When to Use
-
-- Playwright hits a login page that requires human credentials
-- A website shows a CAPTCHA or anti-bot challenge
-- Two-factor authentication (2FA/MFA) requires a code from the user
-- You need the user to visually verify something in the browser
-- Any browser automation step that needs human-in-the-loop interaction
-
-## Parameters
-
-| Parameter | Required | Description |
-|-----------|----------|-------------|
-| `action`  | Yes      | One of: `start`, `stop`, `status`, `get-url` |
-
-### Actions
-
-- **start** — Check Screen Sharing is on, install deps if needed, launch websockify + noVNC + cloudflared. Returns the public URL.
-- **stop** — Shut down websockify + cloudflared. Does NOT touch Screen Sharing.
-- **status** — Check Screen Sharing (port 5900), websockify, and cloudflared status.
-- **get-url** — Retrieve the cloudflared public URL.
-
-## Examples
-
-### Start and get the public URL
-
-```bash
-bash config/skills/agent/vnc-browser/execute.sh '{"action":"start"}'
-```
-
-Output:
-```json
-{
-  "success": true,
-  "status": "started",
-  "publicUrl": "https://abc-xyz.trycloudflare.com/vnc.html?autoconnect=true",
-  "localUrl": "http://localhost:6080/vnc.html?autoconnect=true",
-  "hint": "Share the publicUrl with the user..."
-}
-```
-
-### Send the URL to the user via Slack
-
-```bash
-URL=$(bash config/skills/agent/vnc-browser/execute.sh '{"action":"get-url"}' | jq -r '.publicUrl')
-# Then send via reply-slack or report-status
-```
-
-### Check status
-
-```bash
-bash config/skills/agent/vnc-browser/execute.sh '{"action":"status"}'
-```
-
-### Stop when done
-
-```bash
-bash config/skills/agent/vnc-browser/execute.sh '{"action":"stop"}'
-```
-
-## Workflow
-
-1. **Ensure Screen Sharing is on** (one-time setup)
-2. **Start VNC bridge** → get the public URL
-3. **Launch Playwright** in headful mode (the browser appears on the Mac desktop)
-4. **Navigate** to the page requiring human interaction
-5. **Send the public URL** to the user via Slack
-6. **Wait** for the user to complete the manual step (login, CAPTCHA, etc.)
-7. **Continue automation** once the browser is past the manual step
-8. **Stop VNC bridge** when no longer needed
-
-## Technical Details
-
-- **VNC source**: macOS Screen Sharing on port 5900 (system-managed)
-- **Web client**: noVNC on port 6080 via websockify
-- **Public tunnel**: cloudflared Quick Tunnel (random `*.trycloudflare.com` subdomain)
-- **Security**: Screen Sharing requires macOS user credentials; cloudflared provides HTTPS
-- **Dependencies**: websockify (pip), noVNC (GitHub release), cloudflared (Homebrew) — auto-installed on first run
-- **PID files**: `~/.crewly/vnc/` for websockify and cloudflared lifecycle
-- **Note**: This skill does NOT start or stop macOS Screen Sharing — that is user-managed
-
-## Requirements
-
-- macOS with Screen Sharing enabled
-- Homebrew installed
-- Python 3 with pip (for websockify)
-- Internet access (for cloudflared tunnel and dependency installation)