crewly 1.20.40 → 1.20.47

package/packages/crewly-agent/src/runtime/desktop-checkpoint.ts

@@ -0,0 +1,231 @@
+/**
+ * Checkpoints — how a desktop subgoal is judged done.
+ *
+ * Phase 4 of docs/research/computer-use-capability-assessment.md. The failure
+ * this exists for is the third one in §5.1: the agent says "I saved it" while
+ * the save dialog is still open. Its own account of what happened is exactly
+ * the thing that cannot be trusted, so a subgoal is complete when something
+ * outside the agent says so — a file on disk, an element on screen, a command
+ * that exits zero.
+ *
+ * Checkpoints are deliberately narrow. Each one is a small, decidable
+ * question with a yes or no answer and a reason when it is no, because a
+ * model that is told "the file has 0 bytes" can act, where one told "task
+ * failed" can only guess.
+ *
+ * @module runtime/desktop-checkpoint
+ */
+
+import { spawn } from 'child_process';
+import { promises as fs } from 'fs';
+
+/** How long a shell checkpoint may take before it counts as failed. */
+const SHELL_TIMEOUT_MS = 15_000;
+
+/**
+ * A decidable statement about the world after a subgoal.
+ *
+ * `shell` is the escape hatch, but the named kinds are preferred: they give a
+ * usable reason on failure, where a shell command can only give an exit code.
+ */
+export type Checkpoint =
+  /** A file exists and, unless `allowEmpty`, has content. */
+  | { kind: 'file-exists'; path: string; allowEmpty?: boolean }
+  /** A file exists and contains this text. */
+  | { kind: 'file-contains'; path: string; text: string }
+  /** A file exists and matches this pattern. */
+  | { kind: 'file-matches'; path: string; pattern: string }
+  /** A file is gone (a rename's other half, a cleanup). */
+  | { kind: 'file-absent'; path: string }
+  /** This application is in front. */
+  | { kind: 'app-frontmost'; app: string }
+  /** An element with this name (and optionally role) is on screen. */
+  | { kind: 'element-present'; name: string; role?: string; app?: string }
+  /** No element with this name is on screen — a dialog that should be gone. */
+  | { kind: 'element-absent'; name: string; app?: string }
+  /** This text is readable on screen. */
+  | { kind: 'text-on-screen'; text: string }
+  /** This command exits zero. */
+  | { kind: 'shell'; command: string; description?: string };
+
+/** The answer, with enough detail to act on. */
+export interface CheckpointResult {
+  passed: boolean;
+  /** Why not, phrased for the model to act on. Absent when it passed. */
+  reason?: string;
+  /** What was actually observed, when that helps more than prose. */
+  observed?: string;
+}
+
+/** Injectable IO, so the evaluator is testable without a desktop. */
+export interface CheckpointDeps {
+  readFile?: (path: string) => Promise<string>;
+  statFile?: (path: string) => Promise<{ size: number }>;
+  runShell?: (command: string) => Promise<{ code: number; stdout: string; stderr: string }>;
+  /** Elements currently on screen, from a desktop snapshot. */
+  snapshot?: (app?: string) => Promise<Array<{ role: string; name?: string }>>;
+  /** Text currently on screen, from OCR. */
+  screenText?: () => Promise<string[]>;
+  /** The frontmost application's name. */
+  frontmostApp?: () => Promise<string>;
+}
+
+/**
+ * Run a shell command, capturing its outcome rather than throwing.
+ *
+ * @param command - The command
+ * @returns Exit code and output
+ */
+async function defaultShell(command: string): Promise<{ code: number; stdout: string; stderr: string }> {
+  return new Promise((resolve) => {
+    const child = spawn('bash', ['-c', command], { stdio: ['ignore', 'pipe', 'pipe'] });
+    let stdout = '';
+    let stderr = '';
+    const timer = setTimeout(() => {
+      child.kill('SIGKILL');
+      resolve({ code: 124, stdout, stderr: `timed out after ${SHELL_TIMEOUT_MS}ms` });
+    }, SHELL_TIMEOUT_MS);
+    child.stdout.on('data', (c) => { stdout += String(c); });
+    child.stderr.on('data', (c) => { stderr += String(c); });
+    child.on('error', (err) => { clearTimeout(timer); resolve({ code: 127, stdout, stderr: err.message }); });
+    child.on('close', (code) => { clearTimeout(timer); resolve({ code: code ?? 1, stdout, stderr }); });
+  });
+}
+
+/**
+ * Describe a checkpoint in words, for a plan the owner might read.
+ *
+ * @param checkpoint - The checkpoint
+ * @returns One line
+ *
+ * @example
+ * describeCheckpoint({ kind: 'file-contains', path: '/tmp/a.txt', text: 'ok' })
+ * // → '/tmp/a.txt contains "ok"'
+ */
+export function describeCheckpoint(checkpoint: Checkpoint): string {
+  switch (checkpoint.kind) {
+    case 'file-exists': return `${checkpoint.path} exists${checkpoint.allowEmpty ? '' : ' and is not empty'}`;
+    case 'file-contains': return `${checkpoint.path} contains "${checkpoint.text}"`;
+    case 'file-matches': return `${checkpoint.path} matches /${checkpoint.pattern}/`;
+    case 'file-absent': return `${checkpoint.path} is gone`;
+    case 'app-frontmost': return `${checkpoint.app} is in front`;
+    case 'element-present': return `"${checkpoint.name}" is on screen`;
+    case 'element-absent': return `"${checkpoint.name}" is no longer on screen`;
+    case 'text-on-screen': return `"${checkpoint.text}" is readable on screen`;
+    case 'shell': return checkpoint.description ?? `\`${checkpoint.command}\` succeeds`;
+  }
+}
+
+/**
+ * Decide whether a checkpoint holds.
+ *
+ * Never throws: an unreadable file or a crashed command is a failed
+ * checkpoint with a reason, not an exception for the caller to interpret.
+ *
+ * @param checkpoint - What to check
+ * @param deps - Injected IO
+ * @returns Whether it holds, and why not when it does not
+ *
+ * @example
+ * await evaluateCheckpoint({ kind: 'file-contains', path: '/tmp/a', text: 'hi' })
+ * // → { passed: false, reason: '/tmp/a exists but does not contain "hi"', observed: '…' }
+ */
+export async function evaluateCheckpoint(
+  checkpoint: Checkpoint,
+  deps: CheckpointDeps = {},
+): Promise<CheckpointResult> {
+  const readFile = deps.readFile ?? ((p: string) => fs.readFile(p, 'utf8'));
+  const statFile = deps.statFile ?? (async (p: string) => ({ size: (await fs.stat(p)).size }));
+  const runShell = deps.runShell ?? defaultShell;
+
+  try {
+    switch (checkpoint.kind) {
+      case 'file-exists': {
+        const stat = await statFile(checkpoint.path).catch(() => null);
+        if (!stat) return { passed: false, reason: `${checkpoint.path} does not exist.` };
+        if (!checkpoint.allowEmpty && stat.size === 0) {
+          // An empty file is the signature of a save that opened a dialog and
+          // never completed — worth calling out rather than passing.
+          return { passed: false, reason: `${checkpoint.path} exists but is empty — the write did not complete.` };
+        }
+        return { passed: true };
+      }
+
+      case 'file-absent': {
+        const stat = await statFile(checkpoint.path).catch(() => null);
+        return stat
+          ? { passed: false, reason: `${checkpoint.path} is still there.` }
+          : { passed: true };
+      }
+
+      case 'file-contains':
+      case 'file-matches': {
+        const content = await readFile(checkpoint.path).catch(() => null);
+        if (content === null) return { passed: false, reason: `${checkpoint.path} does not exist or cannot be read.` };
+        const hit = checkpoint.kind === 'file-contains'
+          ? content.includes(checkpoint.text)
+          : new RegExp(checkpoint.pattern).test(content);
+        if (hit) return { passed: true };
+        const wanted = checkpoint.kind === 'file-contains' ? `"${checkpoint.text}"` : `/${checkpoint.pattern}/`;
+        return {
+          passed: false,
+          reason: `${checkpoint.path} exists but does not match ${wanted}.`,
+          observed: content.slice(0, 200),
+        };
+      }
+
+      case 'app-frontmost': {
+        if (!deps.frontmostApp) return { passed: false, reason: 'Cannot see which application is in front.' };
+        const front = await deps.frontmostApp();
+        return front.toLowerCase() === checkpoint.app.toLowerCase()
+          ? { passed: true }
+          : { passed: false, reason: `${checkpoint.app} is not in front.`, observed: front };
+      }
+
+      case 'element-present':
+      case 'element-absent': {
+        if (!deps.snapshot) return { passed: false, reason: 'Cannot read the elements on screen.' };
+        const elements = await deps.snapshot(checkpoint.app);
+        const wanted = checkpoint.name.toLowerCase();
+        const found = elements.find((e) => {
+          if (!e.name || !e.name.toLowerCase().includes(wanted)) return false;
+          return checkpoint.kind === 'element-present' && checkpoint.role ? e.role === checkpoint.role : true;
+        });
+        if (checkpoint.kind === 'element-present') {
+          return found
+            ? { passed: true }
+            : { passed: false, reason: `Nothing called "${checkpoint.name}" is on screen.` };
+        }
+        return found
+          ? { passed: false, reason: `"${checkpoint.name}" is still on screen.`, observed: found.role }
+          : { passed: true };
+      }
+
+      case 'text-on-screen': {
+        if (!deps.screenText) return { passed: false, reason: 'Cannot read the text on screen.' };
+        const lines = await deps.screenText();
+        const wanted = checkpoint.text.toLowerCase();
+        return lines.some((l) => l.toLowerCase().includes(wanted))
+          ? { passed: true }
+          : { passed: false, reason: `"${checkpoint.text}" is not readable on screen.` };
+      }
+
+      case 'shell': {
+        const { code, stdout, stderr } = await runShell(checkpoint.command);
+        if (code === 0) return { passed: true };
+        return {
+          passed: false,
+          reason: `${describeCheckpoint(checkpoint)} — exited ${code}.`,
+          observed: (stderr || stdout).slice(0, 200),
+        };
+      }
+    }
+  } catch (err) {
+    // A checkpoint that throws is a checkpoint that did not pass. Turning it
+    // into an exception would let a broken check read as a broken task.
+    return {
+      passed: false,
+      reason: `Could not check: ${err instanceof Error ? err.message : String(err)}`,
+    };
+  }
+}

package/packages/crewly-agent/src/runtime/desktop-recovery.test.ts

@@ -0,0 +1,100 @@
+/**
+ * Tests for surprise detection.
+ *
+ * The failure being guarded against: a dialog appears at step 15, the agent
+ * does not notice, and the next twenty steps act on a world that no longer
+ * exists. Each one looks fine on its own.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { detectSurprise, shouldRetryAfter, type Scene } from './desktop-recovery.js';
+
+describe('detectSurprise', () => {
+  it('sees nothing wrong with an ordinary window', () => {
+    expect(detectSurprise({ app: 'TextEdit', elements: [{ role: 'AXButton', name: 'Bold' }] })).toBeNull();
+  });
+
+  it('spots a modal and lists the buttons so the agent can choose', () => {
+    const out = detectSurprise({
+      elements: [
+        { role: 'AXSheet', name: 'Save changes?' },
+        { role: 'AXButton', name: 'Save' },
+        { role: 'AXButton', name: "Don't Save" },
+      ],
+    });
+    expect(out?.kind).toBe('modal-dialog');
+    expect(out?.instruction).toContain("Don't Save");
+    // The most common way to get this wrong is to save when nobody asked.
+    expect(out?.instruction).toMatch(/if the task did not ask to save, do not save/i);
+    expect(out?.selfRecoverable).toBe(true);
+  });
+
+  it('tells the agent its refs are stale after a dialog', () => {
+    const out = detectSurprise({ elements: [{ role: 'AXDialog', name: 'Export' }] });
+    expect(out?.instruction).toMatch(/fresh snapshot/);
+  });
+
+  it('treats a permission prompt as the owner\'s decision, not a dialog to answer', () => {
+    const out = detectSurprise({
+      elements: [
+        { role: 'AXSheet', name: 'Terminal would like to access your Documents folder' },
+        { role: 'AXButton', name: 'Allow' },
+      ],
+    });
+    expect(out?.kind).toBe('permission-prompt');
+    expect(out?.selfRecoverable).toBe(false);
+  });
+
+  it('stops at a sign-in wall instead of typing credentials', () => {
+    for (const name of ['Sign in to continue', 'Enter your password', '请输入验证码']) {
+      const out = detectSurprise({ elements: [{ role: 'AXStaticText', name }] });
+      expect(out?.kind, name).toBe('login-required');
+      expect(out?.selfRecoverable).toBe(false);
+    }
+  });
+
+  it('reads a locked screen off the last failure, and calls it unrecoverable', () => {
+    const out = detectSurprise({ elements: [], lastFailure: { reason: 'screen_locked' } });
+    expect(out?.kind).toBe('screen-locked');
+    expect(out?.selfRecoverable).toBe(false);
+  });
+
+  it('warns not to repeat a timed-out action blindly — it may have gone through', () => {
+    const out = detectSurprise({ elements: [], lastFailure: { reason: 'timeout' } });
+    expect(out?.kind).toBe('app-not-responding');
+    expect(out?.instruction).toMatch(/may have gone through/);
+  });
+
+  it('notices focus moving away from the app the plan is about', () => {
+    expect(detectSurprise({ app: 'Safari', elements: [] }, 'TextEdit')?.kind).toBe('focus-lost');
+    expect(detectSurprise({ app: 'TextEdit', elements: [] }, 'textedit')).toBeNull();
+  });
+
+  it('prefers the specific diagnosis when two apply', () => {
+    // A permission prompt is also a modal; the specific instruction is the
+    // useful one.
+    const out = detectSurprise({
+      elements: [
+        { role: 'AXSheet', name: 'Crewly would like to access your Calendar' },
+        { role: 'AXButton', name: 'Allow' },
+      ],
+    });
+    expect(out?.kind).toBe('permission-prompt');
+  });
+});
+
+describe('shouldRetryAfter', () => {
+  it('never retries something that needs a person', () => {
+    const out = shouldRetryAfter('login-required', 0, false);
+    expect(out.retry).toBe(false);
+    expect(out.escalation).toMatch(/needs a person/);
+  });
+
+  it('allows three goes at a recoverable surprise, then stops', () => {
+    expect(shouldRetryAfter('modal-dialog', 0, true).retry).toBe(true);
+    expect(shouldRetryAfter('modal-dialog', 2, true).retry).toBe(true);
+    const out = shouldRetryAfter('modal-dialog', 3, true);
+    expect(out.retry).toBe(false);
+    expect(out.escalation).toMatch(/will not be different/);
+  });
+});

package/packages/crewly-agent/src/runtime/desktop-recovery.ts

@@ -0,0 +1,195 @@
+/**
+ * Surprises, and what to do about them.
+ *
+ * Phase 4 of docs/research/computer-use-capability-assessment.md, §5.7. The
+ * second failure in §5.1 is the expensive one: at step 15 a dialog appears,
+ * the agent does not notice, and the next twenty steps act on a state that no
+ * longer exists. Every one of them looks fine in isolation.
+ *
+ * The surprises are not open-ended. A desktop throws the same handful over
+ * and over — a modal sheet, a permission prompt, a beachballing app, focus
+ * landing somewhere else, a login wall. Naming them means the agent gets a
+ * specific instruction instead of having to work out from a screenshot that
+ * something is wrong at all.
+ *
+ * @module runtime/desktop-recovery
+ */
+
+/** What went unexpectedly wrong. */
+export type SurpriseKind =
+  /** A sheet or modal is waiting for an answer. */
+  | 'modal-dialog'
+  /** macOS is asking the user to allow something. */
+  | 'permission-prompt'
+  /** A sign-in wall. */
+  | 'login-required'
+  /** The app stopped responding. */
+  | 'app-not-responding'
+  /** Another application took the front. */
+  | 'focus-lost'
+  /** The screen was locked mid-task. */
+  | 'screen-locked';
+
+/** A surprise, and the instruction that goes with it. */
+export interface Surprise {
+  kind: SurpriseKind;
+  /** What was seen, for the log and for the agent. */
+  detail: string;
+  /** What the agent should do next, in one instruction. */
+  instruction: string;
+  /**
+   * Whether the agent can deal with this itself. False means a person has to
+   * — there is no point spending the budget discovering that.
+   */
+  selfRecoverable: boolean;
+}
+
+/** An element as a snapshot reports it. */
+export interface SceneElement {
+  role: string;
+  name?: string;
+  subrole?: string;
+}
+
+/** Enough of the world to spot a surprise in. */
+export interface Scene {
+  app?: string;
+  elements: SceneElement[];
+  /** Whatever the last action answered, when it failed. */
+  lastFailure?: { reason?: string; message?: string };
+}
+
+/** Button labels that mean "a modal is waiting". */
+const MODAL_ROLES = new Set(['AXSheet', 'AXDialog']);
+
+/** Words that mark a sign-in wall rather than an ordinary form. */
+const LOGIN_WORDS = ['sign in', 'log in', 'login', 'password', 'two-factor', 'verification code', '验证码', '登录'];
+
+/** Words macOS uses when it wants the user to allow something. */
+const PERMISSION_WORDS = ['would like to access', 'wants access', 'allow', 'grant access', '访问'];
+
+/**
+ * Look at a scene and name what is wrong with it, if anything.
+ *
+ * Order matters: the checks run from most specific to least, because a
+ * permission prompt *is* a modal dialog and the specific instruction is the
+ * useful one.
+ *
+ * @param scene - What is on screen, plus the last failure if there was one
+ * @param expectedApp - The app the agent believes it is working in
+ * @returns The surprise, or null when the scene looks ordinary
+ *
+ * @example
+ * detectSurprise({ app: 'TextEdit', elements: [{ role: 'AXSheet', name: 'Save changes?' }] })
+ * // → { kind: 'modal-dialog', instruction: 'Answer the dialog…' }
+ */
+export function detectSurprise(scene: Scene, expectedApp?: string): Surprise | null {
+  // A rail already said what was wrong; it outranks anything inferred.
+  const failure = scene.lastFailure?.reason;
+  if (failure === 'screen_locked') {
+    return {
+      kind: 'screen-locked',
+      detail: 'The screen was locked.',
+      instruction: 'Stop and report that the screen is locked. It cannot be unlocked from here.',
+      selfRecoverable: false,
+    };
+  }
+
+  const named = scene.elements.filter((e) => e.name);
+  const textOf = (e: SceneElement) => (e.name ?? '').toLowerCase();
+
+  // A permission prompt is a modal, so it has to be checked first.
+  const permission = named.find((e) => PERMISSION_WORDS.some((w) => textOf(e).includes(w)));
+  if (permission && named.some((e) => MODAL_ROLES.has(e.role))) {
+    return {
+      kind: 'permission-prompt',
+      detail: `macOS is asking: "${permission.name}"`,
+      instruction:
+        'A macOS permission prompt is open. Do not answer it — granting access on the owner\'s behalf is their decision. ' +
+        'Stop and tell them what is being asked for.',
+      selfRecoverable: false,
+    };
+  }
+
+  const login = named.find((e) => LOGIN_WORDS.some((w) => textOf(e).includes(w)));
+  if (login) {
+    return {
+      kind: 'login-required',
+      detail: `A sign-in is being asked for: "${login.name}"`,
+      instruction:
+        'This needs credentials, which desktop control never enters. Stop and ask the owner to sign in, then continue.',
+      selfRecoverable: false,
+    };
+  }
+
+  const modal = scene.elements.find((e) => MODAL_ROLES.has(e.role));
+  if (modal) {
+    const buttons = named.filter((e) => e.role === 'AXButton').map((e) => e.name!);
+    return {
+      kind: 'modal-dialog',
+      detail: `A dialog is open${modal.name ? `: "${modal.name}"` : ''}.`,
+      instruction:
+        `A dialog is waiting for an answer and nothing else will work until it is dealt with. ` +
+        (buttons.length
+          ? `Its buttons are: ${buttons.join(', ')}. Choose the one that matches the task — if the task did not ask to save, do not save. `
+          : 'Take a snapshot to see its buttons. ') +
+        'Then take a fresh snapshot: the window behind it has probably changed.',
+      selfRecoverable: true,
+    };
+  }
+
+  if (scene.lastFailure?.reason === 'timeout') {
+    return {
+      kind: 'app-not-responding',
+      detail: 'The last action timed out.',
+      instruction:
+        'The application is not responding. Wait for the screen to settle, then take a fresh snapshot before trying again. ' +
+        'Do not repeat the action blindly — it may have gone through.',
+      selfRecoverable: true,
+    };
+  }
+
+  if (expectedApp && scene.app && scene.app.toLowerCase() !== expectedApp.toLowerCase()) {
+    return {
+      kind: 'focus-lost',
+      detail: `${scene.app} is in front, not ${expectedApp}.`,
+      instruction:
+        `Focus moved to ${scene.app}. Bring ${expectedApp} back to the front and take a fresh snapshot — ` +
+        'any refs from before are stale.',
+      selfRecoverable: true,
+    };
+  }
+
+  return null;
+}
+
+/**
+ * Whether to keep trying after a surprise.
+ *
+ * Three attempts at the same kind is the limit. A desktop surprise that
+ * survives three goes is not one more click away — it is something the agent
+ * has misunderstood, and the cheapest next move is to say so.
+ *
+ * @param kind - What went wrong
+ * @param alreadyTried - How many times this kind has been handled in this subgoal
+ * @param selfRecoverable - From the surprise
+ * @returns Whether to try again, and what to say when not
+ */
+export function shouldRetryAfter(
+  kind: SurpriseKind,
+  alreadyTried: number,
+  selfRecoverable: boolean,
+): { retry: boolean; escalation?: string } {
+  if (!selfRecoverable) {
+    return { retry: false, escalation: `${kind} needs a person — stop and report it rather than working around it.` };
+  }
+  if (alreadyTried >= 3) {
+    return {
+      retry: false,
+      escalation:
+        `The same problem (${kind}) has come back three times. Stop and report what you tried and what you see — ` +
+        'a fourth attempt will not be different.',
+    };
+  }
+  return { retry: true };
+}