crewly 1.20.35 → 1.20.47

package/packages/crewly-agent/src/runtime/desktop-task.tool.test.ts

@@ -0,0 +1,218 @@
+/**
+ * Tests for the `desktop_task` tool.
+ *
+ * Phase 4 built the machinery; this is what connects it, and the connection
+ * is where it can be got wrong. The property under test throughout: an agent
+ * cannot mark its own work done, and a turn that ends with unverified work
+ * says so.
+ */
+
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import {
+  createDesktopTaskTool, resetDesktopTasks, answerDesktopConfirmation, unverifiedDesktopWork,
+} from './desktop-task.tool.js';
+
+const SESSION = 'test-agent';
+
+/** A perceive stub backed by a scriptable scene. */
+function perceiver(scene: Record<string, unknown> = { success: true, app: 'TextEdit', elements: [] }) {
+  return vi.fn(async () => scene);
+}
+
+/** A two-subgoal plan checked against shell commands we control. */
+function plan(overrides: Record<string, unknown> = {}) {
+  return {
+    operation: 'plan' as const,
+    goal: 'write and check a file',
+    subgoals: [
+      { goal: 'write the note', checkpoint: { kind: 'shell' as const, command: 'test -f /tmp/crewly-tt-a' } },
+      { goal: 'clean up', checkpoint: { kind: 'file-absent' as const, path: '/tmp/crewly-tt-a' } },
+    ],
+    ...overrides,
+  };
+}
+
+describe('planning', () => {
+  beforeEach(resetDesktopTasks);
+
+  it('refuses a plan with no subgoals — a task with no checkpoints proves nothing', async () => {
+    const tool = createDesktopTaskTool(SESSION);
+    expect(await tool.execute({ operation: 'plan', goal: 'do it' })).toMatchObject({ reason: 'validation' });
+  });
+
+  it('advises a route per subgoal, while it can still change what the agent does', async () => {
+    const tool = createDesktopTaskTool(SESSION, { availableSkills: ['gmail-send'] });
+    const out = (await tool.execute({
+      operation: 'plan',
+      goal: 'summarise and send',
+      subgoals: [
+        { goal: 'read the report in Preview', checkpoint: { kind: 'app-frontmost', app: 'Preview' } },
+        { goal: 'send it by email', checkpoint: { kind: 'shell', command: 'true' } },
+      ],
+    })) as { routes: Array<Record<string, unknown>> };
+    // The point of routing: not opening Mail.app for something a skill does.
+    expect(out.routes[1]).toMatchObject({ route: 'skill', use: 'gmail-send' });
+  });
+
+  it('will not step before a plan exists', async () => {
+    const tool = createDesktopTaskTool(SESSION);
+    expect(await tool.execute({ operation: 'step' })).toMatchObject({ reason: 'no_task' });
+  });
+});
+
+describe('the agent cannot declare itself done', () => {
+  beforeEach(resetDesktopTasks);
+
+  it('holds the subgoal until the checkpoint holds, and says what is wrong', async () => {
+    const tool = createDesktopTaskTool(SESSION, { perceive: perceiver() });
+    await tool.execute(plan());
+
+    const first = (await tool.execute({ operation: 'step' })) as Record<string, unknown>;
+    expect(first['directive']).toBe('continue');
+    expect(String(first['hint'])).toContain('test -f');
+
+    // Now make it true.
+    const { writeFileSync, unlinkSync } = await import('fs');
+    writeFileSync('/tmp/crewly-tt-a', 'x');
+    try {
+      const second = (await tool.execute({ operation: 'step' })) as Record<string, unknown>;
+      expect(second['directive']).toBe('advance');
+      expect(second['verified']).toBe('write the note');
+    } finally {
+      unlinkSync('/tmp/crewly-tt-a');
+    }
+  });
+
+  it('reports outstanding work as unverified rather than rounding it up', async () => {
+    const tool = createDesktopTaskTool(SESSION, { perceive: perceiver() });
+    await tool.execute(plan());
+    const out = (await tool.execute({ operation: 'summary' })) as Record<string, unknown>;
+    expect(out['complete']).toBe(false);
+    expect(out['unverified']).toEqual(['write the note', 'clean up']);
+  });
+
+  it('makes the whole turn incomplete while a subgoal is unverified', async () => {
+    const tool = createDesktopTaskTool(SESSION, { perceive: perceiver() });
+    await tool.execute(plan());
+    // This is what the runner asks before letting a reply go out. A model
+    // that stopped because it believed it was finished is exactly the case.
+    const outstanding = unverifiedDesktopWork(SESSION);
+    expect(outstanding?.goals).toContain('write the note');
+  });
+
+  it('reports nothing outstanding once every checkpoint held', async () => {
+    const tool = createDesktopTaskTool(SESSION, { perceive: perceiver() });
+    await tool.execute({
+      operation: 'plan',
+      goal: 'trivial',
+      subgoals: [{ goal: 'nothing to do', checkpoint: { kind: 'shell', command: 'true' } }],
+    });
+    await tool.execute({ operation: 'step' });
+    expect(unverifiedDesktopWork(SESSION)).toBeNull();
+  });
+
+  it('knows nothing about a session that never planned anything', () => {
+    expect(unverifiedDesktopWork('never-planned')).toBeNull();
+  });
+});
+
+describe('irreversible actions', () => {
+  beforeEach(resetDesktopTasks);
+
+  it('sends one to the owner and tells the agent to stop, workarounds included', async () => {
+    const enqueue = vi.fn(() => ({ id: 'ap-1' }) as never);
+    const tool = createDesktopTaskTool(SESSION, { perceive: perceiver(), approvals: { enqueue } });
+    await tool.execute(plan());
+
+    const out = (await tool.execute({ operation: 'confirm', action: 'click Send on the email' })) as Record<string, unknown>;
+    expect(out).toMatchObject({ required: true, approvalId: 'ap-1' });
+    expect(String(out['message'])).toMatch(/not a workaround/i);
+    expect(enqueue).toHaveBeenCalledWith(SESSION, 'desktop_task', 'destructive', expect.objectContaining({
+      action: 'click Send on the email',
+    }));
+  });
+
+  it('holds every later step until the owner answers', async () => {
+    const tool = createDesktopTaskTool(SESSION, { perceive: perceiver() });
+    await tool.execute({
+      operation: 'plan', goal: 'send it',
+      // A checkpoint that would pass immediately, to prove the wait wins.
+      subgoals: [{ goal: 'send', checkpoint: { kind: 'shell', command: 'true' } }],
+    });
+    await tool.execute({ operation: 'confirm', action: 'delete the folder' });
+    const out = (await tool.execute({ operation: 'step' })) as Record<string, unknown>;
+    expect(out['directive']).toBe('await-confirmation');
+  });
+
+  it('says so plainly when something is reversible, instead of asking anyway', async () => {
+    const tool = createDesktopTaskTool(SESSION, { perceive: perceiver() });
+    await tool.execute(plan());
+    const out = (await tool.execute({ operation: 'confirm', action: 'open the document' })) as Record<string, unknown>;
+    // An agent that asks about everything learns nothing about where the line is.
+    expect(out).toMatchObject({ required: false });
+  });
+
+  it('carries on when the owner approves', async () => {
+    const tool = createDesktopTaskTool(SESSION, { perceive: perceiver() });
+    await tool.execute(plan());
+    await tool.execute({ operation: 'confirm', action: 'publish the post' });
+    const out = answerDesktopConfirmation(SESSION, true);
+    expect(out).toMatchObject({ directive: 'continue' });
+  });
+
+  it('stops for good when the owner declines', async () => {
+    const tool = createDesktopTaskTool(SESSION, { perceive: perceiver() });
+    await tool.execute(plan());
+    await tool.execute({ operation: 'confirm', action: 'publish the post' });
+    const out = answerDesktopConfirmation(SESSION, false) as Record<string, unknown>;
+    expect(out['directive']).toBe('escalate');
+    expect(String(out['message'])).toMatch(/do not look for another way/i);
+  });
+
+  it('answers nothing when nothing was waiting', () => {
+    expect(answerDesktopConfirmation('idle-session', true)).toBeNull();
+  });
+});
+
+describe('surprises reach the agent', () => {
+  beforeEach(resetDesktopTasks);
+
+  it('passes the dialog instruction through instead of testing the checkpoint', async () => {
+    const perceive = perceiver({
+      success: true, app: 'TextEdit',
+      elements: [
+        { role: 'AXSheet', name: 'Save changes?' },
+        { role: 'AXButton', name: "Don't Save" },
+      ],
+    });
+    const tool = createDesktopTaskTool(SESSION, { perceive });
+    await tool.execute(plan());
+    const out = (await tool.execute({ operation: 'step' })) as Record<string, unknown>;
+    expect(out['directive']).toBe('continue');
+    expect(String(out['hint'])).toContain("Don't Save");
+  });
+
+  it('treats a failed snapshot as an empty scene rather than crashing the step', async () => {
+    const perceive = vi.fn(async () => ({ success: false, reason: 'screen_locked', message: 'locked' }));
+    const tool = createDesktopTaskTool(SESSION, { perceive });
+    await tool.execute(plan());
+    const out = (await tool.execute({ operation: 'step' })) as Record<string, unknown>;
+    // screen_locked is a surprise the runtime knows, and it needs a person.
+    expect(out['directive']).toBe('escalate');
+    expect(out['reason']).toBe('screen-locked');
+  });
+});
+
+describe('abandoning', () => {
+  beforeEach(resetDesktopTasks);
+
+  it('drops the task but still hands back what was and was not done', async () => {
+    const tool = createDesktopTaskTool(SESSION, { perceive: perceiver() });
+    await tool.execute(plan());
+    const out = (await tool.execute({ operation: 'abandon', reason: 'the app is not installed' })) as Record<string, unknown>;
+    expect(String(out['message'])).toContain('the app is not installed');
+    expect(out['summary']).toContain('write the note');
+    // And the turn is no longer held open by a task nobody is working on.
+    expect(unverifiedDesktopWork(SESSION)).toBeNull();
+  });
+});

package/packages/crewly-agent/src/runtime/desktop-task.tool.ts

@@ -0,0 +1,343 @@
+/**
+ * `desktop_task` tool — the task runtime, plugged in.
+ *
+ * Phase 4 built the machinery that decides when a desktop subgoal is really
+ * done; this is what connects it to an agent. Without it the runtime was a
+ * tested library nothing called, which is the least useful state for a piece
+ * of safety machinery to be in.
+ *
+ * Three things make it hard to bypass rather than merely available:
+ *
+ *   - `step` is the only way to advance. An agent cannot mark a subgoal done;
+ *     it asks, and the checkpoint answers.
+ *   - `summary` reports unverified subgoals as unverified. A turn that ends
+ *     with work outstanding says so in the same words the owner would use.
+ *   - An irreversible action goes to the approval queue that already exists,
+ *     so the owner answers it wherever they already answer approvals.
+ *
+ * The tool holds one task per agent session. A desktop has one mouse, so an
+ * agent running two desktop tasks at once is a bug rather than a use case.
+ *
+ * @module runtime/desktop-task.tool
+ */
+
+import { z } from 'zod';
+import type { ToolDefinition } from './types.js';
+import type { ApprovalQueueService } from './approval-queue.service.js';
+import {
+  beginTask, step as runtimeStep, summarize, activeSubgoal, chooseRoute,
+  needsConfirmation, awaitConfirmation, resolveConfirmation,
+  DEFAULT_TASK_BUDGET, type Subgoal, type TaskState, type Directive,
+} from './desktop-task-runtime.js';
+import type { Checkpoint, CheckpointDeps } from './desktop-checkpoint.js';
+import type { Scene, SceneElement } from './desktop-recovery.js';
+
+/** What the tool needs from the outside world. */
+export interface DesktopTaskDeps {
+  /** Run a computer-use action; used to read the scene and test checkpoints. */
+  perceive?: (input: Record<string, unknown>) => Promise<Record<string, unknown>>;
+  /** Where an irreversible action goes for the owner to answer. */
+  approvals?: Pick<ApprovalQueueService, 'enqueue'>;
+  /** Skills this install has, so the router can prefer one over the screen. */
+  availableSkills?: string[];
+  now?: () => number;
+}
+
+/** One task per session: a desktop has one mouse. */
+const tasks = new Map<string, TaskState>();
+
+/** Approval ids waiting on the owner, by session. */
+const pendingApprovals = new Map<string, string>();
+
+/** Drop a session's task (tests, and session teardown). */
+export function resetDesktopTasks(): void {
+  tasks.clear();
+  pendingApprovals.clear();
+}
+
+/** The checkpoint shapes an agent may declare. Mirrors {@link Checkpoint}. */
+const checkpointSchema = z.union([
+  z.object({ kind: z.literal('file-exists'), path: z.string(), allowEmpty: z.boolean().optional() }),
+  z.object({ kind: z.literal('file-contains'), path: z.string(), text: z.string() }),
+  z.object({ kind: z.literal('file-matches'), path: z.string(), pattern: z.string() }),
+  z.object({ kind: z.literal('file-absent'), path: z.string() }),
+  z.object({ kind: z.literal('app-frontmost'), app: z.string() }),
+  z.object({ kind: z.literal('element-present'), name: z.string(), role: z.string().optional(), app: z.string().optional() }),
+  z.object({ kind: z.literal('element-absent'), name: z.string(), app: z.string().optional() }),
+  z.object({ kind: z.literal('text-on-screen'), text: z.string() }),
+  z.object({ kind: z.literal('shell'), command: z.string(), description: z.string().optional() }),
+]);
+
+const schema = z.object({
+  operation: z.enum(['plan', 'step', 'confirm', 'summary', 'abandon'])
+    .describe('plan a task, take a step, ask the owner about something irreversible, report, or give up.'),
+  goal: z.string().optional().describe('The whole task, for `plan`.'),
+  subgoals: z.array(z.object({
+    goal: z.string().describe('What to achieve, in your own words.'),
+    checkpoint: checkpointSchema.describe('How the runtime will know it happened — not your say-so.'),
+    maxSteps: z.number().optional(),
+  })).optional().describe('The plan, for `plan`. Three to six subgoals is usually right.'),
+  action: z.string().optional().describe('For `confirm`: the irreversible thing you are about to do.'),
+  reason: z.string().optional().describe('For `abandon`: why you are stopping.'),
+});
+
+/**
+ * Turn a directive into something a model can act on without re-reading the
+ * whole plan every step.
+ *
+ * @param directive - From the runtime
+ * @param state - Current task
+ * @returns The tool result
+ */
+function renderDirective(directive: Directive, state: TaskState): Record<string, unknown> {
+  const base = { success: true, directive: directive.kind };
+  switch (directive.kind) {
+    case 'continue':
+      return {
+        ...base,
+        subgoal: directive.subgoal.goal,
+        stepsLeft: directive.stepsLeft,
+        // The hint is the reason the checkpoint did not hold, or what the
+        // surprise was. It is the actionable part; the goal is just context.
+        ...(directive.hint ? { hint: directive.hint } : {}),
+      };
+    case 'advance':
+      return { ...base, verified: directive.completed.goal, next: directive.next.goal };
+    case 'done':
+      return { ...base, summary: summarize(state), message: directive.summary };
+    case 'escalate':
+      return {
+        ...base, success: false, reason: directive.reason, message: directive.detail,
+        summary: summarize(state),
+      };
+    case 'await-confirmation':
+      return { ...base, waitingFor: directive.action, message: directive.detail };
+    case 'budget-exhausted':
+      return {
+        ...base, success: false, reason: 'budget_exhausted', message: directive.reason,
+        completed: directive.completed, remaining: directive.remaining, summary: summarize(state),
+      };
+  }
+}
+
+/**
+ * Read the screen into the shape the surprise detector wants.
+ *
+ * A failed or missing snapshot yields an empty scene rather than throwing:
+ * not being able to look is itself worth continuing past, and the checkpoint
+ * will fail honestly a moment later.
+ *
+ * @param deps - Injected IO
+ * @returns The scene
+ */
+async function readScene(deps: DesktopTaskDeps): Promise<Scene> {
+  if (!deps.perceive) return { elements: [] };
+  const snap = await deps.perceive({ action: 'snapshot' }).catch(() => null);
+  if (!snap || snap['success'] === false) {
+    return {
+      elements: [],
+      ...(snap ? { lastFailure: { reason: String(snap['reason'] ?? ''), message: String(snap['message'] ?? '') } } : {}),
+    };
+  }
+  const elements = (snap['elements'] as Array<Record<string, unknown>> | undefined) ?? [];
+  return {
+    ...(snap['app'] ? { app: String(snap['app']) } : {}),
+    elements: elements.map((e): SceneElement => ({
+      role: String(e['role'] ?? ''),
+      ...(e['name'] ? { name: String(e['name']) } : {}),
+      ...(e['subrole'] ? { subrole: String(e['subrole']) } : {}),
+    })),
+  };
+}
+
+/** Checkpoint IO backed by the desktop, for the screen-reading kinds. */
+function checkpointDeps(deps: DesktopTaskDeps): CheckpointDeps {
+  if (!deps.perceive) return {};
+  return {
+    snapshot: async (app?: string) => {
+      const snap = await deps.perceive!({ action: 'snapshot', ...(app ? { app } : {}) });
+      const elements = (snap['elements'] as Array<Record<string, unknown>> | undefined) ?? [];
+      return elements.map((e) => ({
+        role: String(e['role'] ?? ''),
+        ...(e['name'] ? { name: String(e['name']) } : {}),
+      }));
+    },
+    screenText: async () => {
+      const out = await deps.perceive!({ action: 'ocr' });
+      const items = (out['items'] as Array<Record<string, unknown>> | undefined) ?? [];
+      return items.map((i) => String(i['text'] ?? ''));
+    },
+    frontmostApp: async () => {
+      const out = await deps.perceive!({ action: 'snapshot' });
+      return String(out['app'] ?? '');
+    },
+  };
+}
+
+/**
+ * Build the `desktop_task` tool.
+ *
+ * @param sessionName - Whose task this is
+ * @param deps - Injected IO
+ * @returns The tool definition
+ */
+export function createDesktopTaskTool(sessionName: string, deps: DesktopTaskDeps = {}): ToolDefinition {
+  const now = deps.now ?? Date.now;
+
+  return {
+    description:
+      'Run a multi-step desktop task so it cannot end in a false "done". ' +
+      'Call `plan` once with subgoals and a checkpoint for each — a checkpoint is something outside you that ' +
+      'proves the subgoal happened (a file with the right contents, a dialog that is gone, an app in front). ' +
+      'Then use the `computer` tool to work, and call `step` after each action: it tells you whether the ' +
+      'checkpoint held, what changed unexpectedly, and when to move on. ' +
+      'Call `confirm` before anything irreversible (sending, deleting, publishing, paying) — the owner answers. ' +
+      'You cannot mark a subgoal done yourself; that is the point.',
+    inputSchema: schema,
+    sensitivity: 'sensitive',
+    execute: async (rawArgs) => {
+      const args = rawArgs as z.infer<typeof schema>;
+      const existing = tasks.get(sessionName);
+
+      switch (args.operation) {
+        case 'plan': {
+          if (!args.goal || !args.subgoals?.length) {
+            return { success: false, reason: 'validation', message: 'plan needs `goal` and at least one subgoal.' };
+          }
+          const subgoals: Subgoal[] = args.subgoals.map((s, i) => ({
+            id: `s${i + 1}`,
+            goal: s.goal,
+            checkpoint: s.checkpoint as Checkpoint,
+            ...(s.maxSteps ? { maxSteps: s.maxSteps } : {}),
+          }));
+          const state = beginTask(args.goal, subgoals, DEFAULT_TASK_BUDGET, now);
+          tasks.set(sessionName, state);
+
+          // The route is advice given once, up front, where it can still
+          // change what the agent does — after it has opened an app it is
+          // too late to say a skill would have been better.
+          const routes = subgoals.map((s) => {
+            const r = chooseRoute(s.goal, deps.availableSkills ?? []);
+            return { subgoal: s.goal, route: r.route, because: r.because, ...(r.candidate ? { use: r.candidate } : {}) };
+          });
+          return {
+            success: true,
+            planned: subgoals.length,
+            first: subgoals[0]!.goal,
+            routes,
+            budget: { steps: state.budget.maxSteps, minutes: Math.round(state.budget.maxDurationMs / 60000) },
+            message: 'Work on the first subgoal, then call step. You cannot skip a checkpoint.',
+          };
+        }
+
+        case 'step': {
+          if (!existing) {
+            return { success: false, reason: 'no_task', message: 'No task planned. Call `plan` first.' };
+          }
+          const scene = await readScene(deps);
+          const directive = await runtimeStep(existing, scene, checkpointDeps(deps), now);
+          if (directive.kind === 'done' || directive.kind === 'escalate' || directive.kind === 'budget-exhausted') {
+            // The task is over either way; keep the state for `summary` but
+            // stop it being stepped again.
+            tasks.set(sessionName, existing);
+          }
+          return renderDirective(directive, existing);
+        }
+
+        case 'confirm': {
+          if (!existing) {
+            return { success: false, reason: 'no_task', message: 'No task planned. Call `plan` first.' };
+          }
+          if (!args.action) {
+            return { success: false, reason: 'validation', message: 'confirm needs `action` — what you are about to do.' };
+          }
+          const verdict = needsConfirmation(args.action);
+          if (!verdict.required) {
+            // Saying so beats silently approving: an agent that asks about
+            // everything learns nothing, and one told "no need" learns where
+            // the line is.
+            return {
+              success: true, required: false,
+              message: `"${args.action}" is reversible — go ahead without asking.`,
+            };
+          }
+          awaitConfirmation(existing, args.action, now);
+          const approval = deps.approvals?.enqueue(sessionName, 'desktop_task', 'destructive', {
+            action: args.action,
+            goal: existing.goal,
+            subgoal: activeSubgoal(existing)?.subgoal.goal ?? '',
+          });
+          if (approval) pendingApprovals.set(sessionName, approval.id);
+          return {
+            success: true,
+            required: true,
+            trigger: verdict.trigger,
+            ...(approval ? { approvalId: approval.id } : {}),
+            message:
+              `Waiting for the owner to approve "${args.action}". Do nothing else until they answer — ` +
+              'not a workaround, not a different route to the same effect.',
+          };
+        }
+
+        case 'summary': {
+          if (!existing) return { success: true, message: 'No task in progress.' };
+          const outstanding = existing.progress.filter((p) => p.state !== 'done');
+          return {
+            success: true,
+            complete: outstanding.length === 0,
+            // Reported as unverified, not as "probably fine". This is the
+            // sentence that stops a turn ending in a confident false report.
+            unverified: outstanding.map((p) => p.subgoal.goal),
+            summary: summarize(existing),
+          };
+        }
+
+        case 'abandon': {
+          tasks.delete(sessionName);
+          pendingApprovals.delete(sessionName);
+          return {
+            success: true,
+            message: `Task dropped${args.reason ? `: ${args.reason}` : ''}. Say what was and was not done.`,
+            ...(existing ? { summary: summarize(existing) } : {}),
+          };
+        }
+      }
+    },
+  };
+}
+
+/**
+ * Record the owner's answer to a pending confirmation.
+ *
+ * Called by whatever surfaces approvals (the REST endpoint, Slack), not by
+ * the agent — which is the whole point of the confirmation.
+ *
+ * @param sessionName - Whose task
+ * @param approved - The owner's decision
+ * @returns What the agent will be told next, or null when nothing was waiting
+ */
+export function answerDesktopConfirmation(sessionName: string, approved: boolean): Record<string, unknown> | null {
+  const state = tasks.get(sessionName);
+  if (!state?.awaitingConfirmation) return null;
+  const directive = resolveConfirmation(state, approved);
+  pendingApprovals.delete(sessionName);
+  return renderDirective(directive, state);
+}
+
+/**
+ * Whether a session has desktop work that never passed its checkpoint.
+ *
+ * The runner asks this at the end of a turn: a task with unverified subgoals
+ * makes the turn incomplete, which is reported to the owner in the same
+ * machinery that already reports a truncated or interrupted turn.
+ *
+ * @param sessionName - Whose task
+ * @returns The unfinished subgoals, or null when there is nothing outstanding
+ */
+export function unverifiedDesktopWork(sessionName: string): { goals: string[]; summary: string } | null {
+  const state = tasks.get(sessionName);
+  if (!state) return null;
+  const outstanding = state.progress.filter((p) => p.state !== 'done' && p.state !== 'skipped');
+  if (outstanding.length === 0) return null;
+  return { goals: outstanding.map((p) => p.subgoal.goal), summary: summarize(state) };
+}