crewly 1.20.40 → 1.20.47

package/packages/crewly-agent/src/runtime/computer.tool.ts

@@ -0,0 +1,405 @@
+/**
+ * `computer` tool — desktop control for the in-process runtime.
+ *
+ * Phase 3 of docs/research/computer-use-capability-assessment.md. The
+ * computer-use skill could already drive the desktop, but an in-process agent
+ * reached it the long way round: `bash_exec` the script, read the JSON, then
+ * `read_file` the screenshot it wrote. Two tool calls per step, and a weak
+ * model had to remember a shell invocation and a file path to get one look at
+ * the screen.
+ *
+ * This is one call that returns the result *and* the new screenshot, which is
+ * what every published computer-use agent expects.
+ *
+ * Two deliberate choices:
+ *
+ * The action names and the coordinate convention match Anthropic's
+ * `computer_20250124` tool. Claude-family models have seen that shape in
+ * training and need no instruction; for every other model there is a large
+ * body of public examples to imitate. Inventing our own names would cost
+ * accuracy for nothing. Crewly's element-level actions (`snapshot`,
+ * `click_ref`, `fill_ref`, `wait_for`) are added alongside — they have no
+ * equivalent in that spec and are the ones a weak model should reach for
+ * first, because naming `@e12` cannot miss the way a coordinate can.
+ *
+ * Nothing here talks to the mouse. Every action shells out to the
+ * computer-use skill, so the safety rails — permissions, stop switch, desktop
+ * lock, destructive-key and password-field refusals, the audit log — apply
+ * exactly once, in one place, to every runtime. A second implementation here
+ * would be a second thing to keep in step, and the rails are the part that
+ * must not drift.
+ *
+ * @module runtime/computer.tool
+ */
+
+import { spawn } from 'child_process';
+import { promises as fs } from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { z } from 'zod';
+import type { ToolDefinition } from './types.js';
+
+/**
+ * Width every screenshot is scaled to before the model sees it.
+ *
+ * Anthropic's guidance, and the reason is worth keeping in mind: accuracy
+ * falls off above roughly this width because the image is downsampled before
+ * the model ever sees it, and a model reasoning in the original coordinate
+ * space then points at the wrong place. The model works in scaled
+ * coordinates; this tool converts them back.
+ */
+const TARGET_WIDTH = 1280;
+
+/** Actions that move or type, and so return a fresh screenshot afterwards. */
+const MUTATING = new Set([
+  'left_click', 'right_click', 'middle_click', 'double_click', 'triple_click',
+  'left_click_drag', 'mouse_move', 'key', 'type', 'scroll', 'click_ref', 'fill_ref',
+]);
+
+/** How long an action may take before the tool gives up on the skill. */
+const DEFAULT_TIMEOUT_MS = 60_000;
+
+/** Injectable IO, for tests. */
+export interface ComputerToolDeps {
+  /** Run the skill and return its stdout. */
+  runSkill?: (input: Record<string, unknown>) => Promise<string>;
+  /** Read a screenshot file as base64. */
+  readImage?: (file: string) => Promise<{ data: string; bytes: number }>;
+  /** Crewly install directory, holding config/skills. */
+  installDir?: string;
+}
+
+/** One screen, as the skill reports it. */
+interface DisplayInfo {
+  frame: [number, number, number, number];
+  scale: number;
+  main: boolean;
+}
+
+/**
+ * Run the computer-use skill with a JSON payload.
+ *
+ * Failures come back as the skill's own JSON where possible: its refusals
+ * (`permission_required`, `screen_locked`, `destructive_blocked`…) already
+ * say what to do about them, and rewording them here would only blur that.
+ *
+ * @param input - The skill's JSON input
+ * @param deps - Injected IO
+ * @returns Parsed skill output
+ */
+async function runSkill(input: Record<string, unknown>, deps: ComputerToolDeps): Promise<Record<string, unknown>> {
+  if (deps.runSkill) {
+    const raw = await deps.runSkill(input);
+    return parseSkillOutput(raw);
+  }
+
+  const installDir = deps.installDir ?? process.env['CREWLY_INSTALL_DIR'] ?? process.cwd();
+  const script = path.join(installDir, 'config', 'skills', 'agent', 'computer-use', 'execute.sh');
+
+  return new Promise((resolve) => {
+    const child = spawn('bash', [script, JSON.stringify(input)], {
+      env: { ...process.env },
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    let stdout = '';
+    let stderr = '';
+    const timer = setTimeout(() => {
+      child.kill('SIGKILL');
+      resolve({
+        success: false,
+        reason: 'timeout',
+        message: `The desktop action did not finish within ${DEFAULT_TIMEOUT_MS / 1000}s.`,
+      });
+    }, DEFAULT_TIMEOUT_MS);
+
+    child.stdout.on('data', (chunk) => { stdout += String(chunk); });
+    child.stderr.on('data', (chunk) => { stderr += String(chunk); });
+    child.on('error', (err) => {
+      clearTimeout(timer);
+      resolve({ success: false, reason: 'skill_unavailable', message: err.message, script });
+    });
+    child.on('close', () => {
+      clearTimeout(timer);
+      resolve(parseSkillOutput(stdout || stderr));
+    });
+  });
+}
+
+/**
+ * Parse the skill's stdout.
+ *
+ * The skill prints one JSON object, but a shell warning can precede it (the
+ * shared runner warns when CREWLY_SESSION_NAME is unset), so the last
+ * JSON-looking line wins.
+ *
+ * @param raw - Captured output
+ * @returns The parsed object, or a described failure when there is none
+ */
+export function parseSkillOutput(raw: string): Record<string, unknown> {
+  const lines = (raw ?? '').trim().split('\n').filter((l) => l.trim());
+  for (let i = lines.length - 1; i >= 0; i--) {
+    const line = lines[i]!.trim();
+    if (!line.startsWith('{')) continue;
+    try {
+      return JSON.parse(line) as Record<string, unknown>;
+    } catch {
+      // Not the JSON line after all — keep looking backwards.
+    }
+  }
+  // Multi-line pretty-printed JSON (jq's default) is one object across lines.
+  const joined = lines.join('\n');
+  const start = joined.indexOf('{');
+  if (start >= 0) {
+    try {
+      return JSON.parse(joined.slice(start)) as Record<string, unknown>;
+    } catch {
+      // Fall through to the described failure.
+    }
+  }
+  return { success: false, reason: 'unparsable', message: raw?.slice(0, 500) || 'The skill produced no output.' };
+}
+
+/**
+ * The scale between the coordinates the model uses and real screen points.
+ *
+ * @param displays - What the skill reported
+ * @returns Factor to multiply model coordinates by, and the scaled size
+ */
+export function scaleFor(displays: DisplayInfo[]): { factor: number; width: number; height: number; screen: [number, number] } {
+  const main = displays.find((d) => d.main) ?? displays[0];
+  const [, , w, h] = main?.frame ?? [0, 0, TARGET_WIDTH, 800];
+  // Never scale up: a small screen is already easier to point at than a large
+  // one, and enlarging it would invent precision the model does not have.
+  const factor = w > TARGET_WIDTH ? w / TARGET_WIDTH : 1;
+  return {
+    factor,
+    width: Math.round(w / factor),
+    height: Math.round(h / factor),
+    screen: [w, h],
+  };
+}
+
+/**
+ * Convert a model coordinate into a screen point.
+ *
+ * @param value - Coordinate in the scaled space the model sees
+ * @param factor - From {@link scaleFor}
+ * @returns Screen point
+ */
+export function toScreen(value: number, factor: number): number {
+  return Math.round(value * factor);
+}
+
+/**
+ * Take a screenshot scaled for the model.
+ *
+ * @param deps - Injected IO
+ * @returns Image payload, or null when the screenshot failed
+ */
+async function capture(
+  deps: ComputerToolDeps,
+): Promise<{ data: string; bytes: number; file: string } | null> {
+  const file = path.join(os.tmpdir(), `crewly-computer-${process.pid}-${Date.now()}.png`);
+  // maxWidth is what makes the screenshot match the coordinate space the
+  // model is told to use; without it the two drift and every click is off.
+  const shot = await runSkill(
+    { action: 'screenshot', output: file, maxWidth: Math.round(TARGET_WIDTH) },
+    deps,
+  );
+  const written = (shot['path'] as string) ?? file;
+  try {
+    if (deps.readImage) {
+      const { data, bytes } = await deps.readImage(written);
+      return { data, bytes, file: written };
+    }
+    const buffer = await fs.readFile(written);
+    await fs.unlink(written).catch(() => undefined);
+    return { data: buffer.toString('base64'), bytes: buffer.length, file: written };
+  } catch {
+    return null;
+  }
+}
+
+/** Arguments the model may send. */
+const computerSchema = z.object({
+  action: z.enum([
+    // Anthropic computer_20250124 vocabulary.
+    'screenshot', 'left_click', 'right_click', 'middle_click', 'double_click',
+    'triple_click', 'left_click_drag', 'mouse_move', 'key', 'type', 'scroll',
+    'wait', 'cursor_position',
+    // Crewly's element-level additions.
+    'snapshot', 'click_ref', 'fill_ref', 'wait_for', 'ocr', 'displays',
+  ]).describe('What to do. Prefer snapshot + click_ref/fill_ref over coordinates when the app exposes elements.'),
+  coordinate: z.array(z.number()).length(2).optional()
+    .describe('[x, y] in the screenshot you were shown, not screen pixels. The tool converts.'),
+  start_coordinate: z.array(z.number()).length(2).optional()
+    .describe('[x, y] to drag from, for left_click_drag.'),
+  text: z.string().optional()
+    .describe('Text to type, the key combo for `key` (e.g. "command+s"), or the value for fill_ref.'),
+  ref: z.string().optional().describe('Element reference from a snapshot, e.g. "@e12".'),
+  app: z.string().optional().describe('Application to snapshot or wait for; defaults to the frontmost.'),
+  scroll_direction: z.enum(['up', 'down', 'left', 'right']).optional(),
+  scroll_amount: z.number().optional().describe('Scroll clicks; defaults to 3.'),
+  duration: z.number().optional().describe('Seconds to wait, for `wait`.'),
+});
+
+/**
+ * Translate the tool's arguments into the skill's own input.
+ *
+ * @param args - Validated tool arguments
+ * @param factor - Coordinate scale
+ * @returns The skill payload, or a refusal when the arguments do not fit
+ */
+export function toSkillInput(
+  args: z.infer<typeof computerSchema>,
+  factor: number,
+): Record<string, unknown> | { error: string } {
+  const point = (pair?: number[]) =>
+    pair ? { x: toScreen(pair[0]!, factor), y: toScreen(pair[1]!, factor) } : null;
+
+  switch (args.action) {
+    case 'screenshot':
+      return { action: 'screenshot' };
+    case 'displays':
+      return { action: 'displays' };
+    case 'cursor_position':
+      // The skill has no cursor read; a screenshot answers the same question
+      // and is what the model will ask for next anyway.
+      return { action: 'screenshot' };
+
+    case 'left_click':
+    case 'right_click':
+    case 'double_click': {
+      const p = point(args.coordinate);
+      if (!p) return { error: `${args.action} needs a coordinate.` };
+      const button = args.action === 'right_click' ? 'right' : args.action === 'double_click' ? 'double' : 'left';
+      return { action: 'click', ...p, button };
+    }
+    case 'middle_click':
+    case 'triple_click': {
+      // Neither exists in the skill. Saying so beats silently doing something
+      // else: a model told "not supported" picks another route, one told
+      // "done" builds on a click that never happened.
+      return { error: `${args.action} is not supported on this platform. Use left_click, or select the text another way.` };
+    }
+    case 'mouse_move': {
+      const p = point(args.coordinate);
+      if (!p) return { error: 'mouse_move needs a coordinate.' };
+      return { action: 'move', ...p };
+    }
+    case 'left_click_drag': {
+      const from = point(args.start_coordinate);
+      const to = point(args.coordinate);
+      if (!from || !to) return { error: 'left_click_drag needs start_coordinate and coordinate.' };
+      return { action: 'drag', fromX: from.x, fromY: from.y, toX: to.x, toY: to.y };
+    }
+    case 'key':
+      if (!args.text) return { error: 'key needs `text`, e.g. "command+s".' };
+      return { action: 'key', key: args.text };
+    case 'type':
+      if (!args.text) return { error: 'type needs `text`.' };
+      return { action: 'type', text: args.text };
+    case 'scroll': {
+      const p = point(args.coordinate);
+      return {
+        action: 'scroll',
+        ...(p ?? {}),
+        direction: args.scroll_direction ?? 'down',
+        amount: args.scroll_amount ?? 3,
+      };
+    }
+
+    case 'snapshot':
+      return { action: 'snapshot', ...(args.app ? { app: args.app } : {}) };
+    case 'click_ref':
+      if (!args.ref) return { error: 'click_ref needs `ref`, e.g. "@e12" from a snapshot.' };
+      return { action: 'click-ref', ref: args.ref };
+    case 'fill_ref':
+      if (!args.ref || args.text === undefined) return { error: 'fill_ref needs `ref` and `text`.' };
+      return { action: 'fill-ref', ref: args.ref, text: args.text };
+    case 'ocr':
+      return { action: 'ocr' };
+    case 'wait_for':
+      if (!args.app && !args.ref && !args.text) {
+        return { error: 'wait_for needs one of `app`, `ref` or `text`.' };
+      }
+      return {
+        action: 'wait-for',
+        ...(args.app ? { app: args.app } : {}),
+        ...(args.ref ? { ref: args.ref } : {}),
+        ...(args.text ? { text: args.text } : {}),
+      };
+    case 'wait':
+      return { action: 'wait-for', idle: true, timeoutMs: Math.round((args.duration ?? 1) * 1000) };
+  }
+}
+
+/**
+ * Build the `computer` tool.
+ *
+ * @param deps - Injected IO for tests
+ * @returns The tool definition
+ *
+ * @example
+ * ```ts
+ * const tools = { computer: createComputerTool() };
+ * ```
+ */
+export function createComputerTool(deps: ComputerToolDeps = {}): ToolDefinition {
+  return {
+    description:
+      'Control this Mac: look at the screen and act on it. ' +
+      'Prefer `snapshot` then `click_ref`/`fill_ref` — naming an element cannot miss the way a coordinate can, ' +
+      'and it keeps working when the window moves. Fall back to coordinates for canvases and custom-drawn UI. ' +
+      'Coordinates are in the screenshot you were shown, not screen pixels. ' +
+      'Destructive key combos, password fields and credential apps are refused, and the owner can stop everything at any time.',
+    inputSchema: computerSchema,
+    sensitivity: 'destructive',
+    execute: async (rawArgs) => {
+      const args = rawArgs as z.infer<typeof computerSchema>;
+
+      // The scale has to come from the live display: the owner may have
+      // changed resolution or moved to another screen since the last call.
+      const displayResult = await runSkill({ action: 'displays' }, deps);
+      const displays = (displayResult['displays'] as DisplayInfo[] | undefined) ?? [];
+      const scale = scaleFor(displays);
+
+      const payload = toSkillInput(args, scale.factor);
+      if ('error' in payload) {
+        return { success: false, reason: 'bad_arguments', message: payload.error };
+      }
+
+      // A bare screenshot is taken once, by the capture step below. Running
+      // the skill's screenshot here as well would shoot the screen twice for
+      // one request — slow, and the two images could even differ.
+      const result = args.action === 'screenshot'
+        ? { success: true }
+        : await runSkill(payload, deps);
+
+      // A refusal is returned as it stands. The rails phrase their own
+      // reasons and tell the agent what to do; a screenshot alongside would
+      // just be the same screen it could not act on.
+      if (result['success'] === false) return result;
+
+      const out: Record<string, unknown> = {
+        ...result,
+        action: args.action,
+        screen: { width: scale.width, height: scale.height, actual: scale.screen },
+      };
+
+      // One call, one look. Showing the result of an action is what lets a
+      // model check its own work instead of assuming the click landed.
+      if (MUTATING.has(args.action) || args.action === 'screenshot') {
+        const image = await capture(deps);
+        if (image) {
+          out['type'] = 'image';
+          out['mimeType'] = 'image/png';
+          out['data'] = image.data;
+          out['sizeBytes'] = image.bytes;
+          out['note'] = `Screenshot is ${scale.width}×${scale.height}; give coordinates in that space.`;
+        }
+      }
+      return out;
+    },
+  };
+}

package/packages/crewly-agent/src/runtime/desktop-checkpoint.test.ts

@@ -0,0 +1,136 @@
+/**
+ * Tests for checkpoint evaluation.
+ *
+ * A checkpoint is the only thing that can call a desktop subgoal done, so a
+ * checkpoint that passes when it should not is worse than having none: it
+ * launders the agent's claim into a verified fact.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { evaluateCheckpoint, describeCheckpoint, type Checkpoint } from './desktop-checkpoint.js';
+
+/** IO over a fake filesystem. */
+function fs(files: Record<string, string>) {
+  return {
+    readFile: async (p: string) => {
+      if (!(p in files)) throw new Error('ENOENT');
+      return files[p]!;
+    },
+    statFile: async (p: string) => {
+      if (!(p in files)) throw new Error('ENOENT');
+      return { size: files[p]!.length };
+    },
+  };
+}
+
+describe('file checkpoints', () => {
+  it('fails an empty file — the signature of a save that never completed', async () => {
+    const out = await evaluateCheckpoint({ kind: 'file-exists', path: '/a' }, fs({ '/a': '' }));
+    expect(out.passed).toBe(false);
+    expect(out.reason).toMatch(/empty/);
+  });
+
+  it('allows an empty file when the task said so', async () => {
+    expect((await evaluateCheckpoint({ kind: 'file-exists', path: '/a', allowEmpty: true }, fs({ '/a': '' }))).passed).toBe(true);
+  });
+
+  it('says the file is missing rather than just failing', async () => {
+    const out = await evaluateCheckpoint({ kind: 'file-exists', path: '/nope' }, fs({}));
+    expect(out.reason).toContain('/nope');
+  });
+
+  it('shows what the file does say when the content is wrong', async () => {
+    const out = await evaluateCheckpoint({ kind: 'file-contains', path: '/a', text: 'hello' }, fs({ '/a': 'goodbye' }));
+    expect(out.passed).toBe(false);
+    expect(out.observed).toBe('goodbye');
+  });
+
+  it('matches a pattern', async () => {
+    expect((await evaluateCheckpoint({ kind: 'file-matches', path: '/a', pattern: '^\\d+$' }, fs({ '/a': '42' }))).passed).toBe(true);
+    expect((await evaluateCheckpoint({ kind: 'file-matches', path: '/a', pattern: '^\\d+$' }, fs({ '/a': 'x' }))).passed).toBe(false);
+  });
+
+  it('checks the other half of a rename', async () => {
+    expect((await evaluateCheckpoint({ kind: 'file-absent', path: '/old' }, fs({}))).passed).toBe(true);
+    expect((await evaluateCheckpoint({ kind: 'file-absent', path: '/old' }, fs({ '/old': 'x' }))).passed).toBe(false);
+  });
+});
+
+describe('screen checkpoints', () => {
+  const snapshot = async () => [
+    { role: 'AXButton', name: 'Save' },
+    { role: 'AXStaticText', name: 'Untitled document' },
+  ];
+
+  it('finds an element by name, and by role when given one', async () => {
+    expect((await evaluateCheckpoint({ kind: 'element-present', name: 'Save' }, { snapshot })).passed).toBe(true);
+    expect((await evaluateCheckpoint({ kind: 'element-present', name: 'Save', role: 'AXButton' }, { snapshot })).passed).toBe(true);
+    expect((await evaluateCheckpoint({ kind: 'element-present', name: 'Save', role: 'AXMenuItem' }, { snapshot })).passed).toBe(false);
+  });
+
+  it('checks a dialog is gone, which is how "closed it" is proved', async () => {
+    expect((await evaluateCheckpoint({ kind: 'element-absent', name: 'Save' }, { snapshot })).passed).toBe(false);
+    expect((await evaluateCheckpoint({ kind: 'element-absent', name: 'Nothing' }, { snapshot })).passed).toBe(true);
+  });
+
+  it('reads text off the screen for apps with no element tree', async () => {
+    const screenText = async () => ['CREWLY EVAL 7734'];
+    expect((await evaluateCheckpoint({ kind: 'text-on-screen', text: 'eval 7734' }, { screenText })).passed).toBe(true);
+    expect((await evaluateCheckpoint({ kind: 'text-on-screen', text: 'absent' }, { screenText })).passed).toBe(false);
+  });
+
+  it('compares the frontmost app case-insensitively and reports what is actually there', async () => {
+    const frontmostApp = async () => 'TextEdit';
+    expect((await evaluateCheckpoint({ kind: 'app-frontmost', app: 'textedit' }, { frontmostApp })).passed).toBe(true);
+    const out = await evaluateCheckpoint({ kind: 'app-frontmost', app: 'Numbers' }, { frontmostApp });
+    expect(out.observed).toBe('TextEdit');
+  });
+
+  it('fails rather than pretends when it cannot see', async () => {
+    // No snapshot dependency wired: it must not quietly pass.
+    expect((await evaluateCheckpoint({ kind: 'element-present', name: 'Save' }, {})).passed).toBe(false);
+    expect((await evaluateCheckpoint({ kind: 'text-on-screen', text: 'x' }, {})).passed).toBe(false);
+    expect((await evaluateCheckpoint({ kind: 'app-frontmost', app: 'x' }, {})).passed).toBe(false);
+  });
+});
+
+describe('shell checkpoints', () => {
+  it('passes on exit zero and reports the output on failure', async () => {
+    const runShell = async (c: string) =>
+      c === 'true' ? { code: 0, stdout: '', stderr: '' } : { code: 1, stdout: '', stderr: 'boom' };
+    expect((await evaluateCheckpoint({ kind: 'shell', command: 'true' }, { runShell })).passed).toBe(true);
+    const out = await evaluateCheckpoint({ kind: 'shell', command: 'false' }, { runShell });
+    expect(out.passed).toBe(false);
+    expect(out.observed).toBe('boom');
+  });
+});
+
+describe('robustness', () => {
+  it('turns a thrown check into a failed one, never an exception', async () => {
+    const out = await evaluateCheckpoint({ kind: 'file-exists', path: '/a' }, {
+      statFile: async () => { throw new Error('disk on fire'); },
+    });
+    // A broken check must not read as a broken task, but it must not pass.
+    expect(out.passed).toBe(false);
+  });
+});
+
+describe('describeCheckpoint', () => {
+  it('reads as a sentence for every kind', () => {
+    const all: Checkpoint[] = [
+      { kind: 'file-exists', path: '/a' },
+      { kind: 'file-contains', path: '/a', text: 'x' },
+      { kind: 'file-matches', path: '/a', pattern: 'x' },
+      { kind: 'file-absent', path: '/a' },
+      { kind: 'app-frontmost', app: 'Finder' },
+      { kind: 'element-present', name: 'Save' },
+      { kind: 'element-absent', name: 'Save' },
+      { kind: 'text-on-screen', text: 'hi' },
+      { kind: 'shell', command: 'true' },
+    ];
+    for (const cp of all) {
+      expect(describeCheckpoint(cp).length).toBeGreaterThan(5);
+    }
+    expect(describeCheckpoint({ kind: 'shell', command: 'x', description: 'the build passes' })).toBe('the build passes');
+  });
+});