@isomorph.ai/app-sdk 1.41.0 → 1.43.0

package/README.md

@@ -448,11 +448,11 @@ export default async function viewComputer(input: { taskId: string; pauseId: str
 
 Keep that table protected by the app's user row rules. A trusted action must still check `owner_id` against `ctx.user.id`; never use workload identity as proof of human ownership.
 
-- `requestTakeover(reason, requiredInput?)` requests a pause; `requiredInput` is `human` (default) or `login`. Poll `check()` until `waiting_for_input` gives `taskId`, `pauseId`, `reason`, `requiredInput` and optional login result.
-- `view(pauseId)` returns `{status:'pausing'}` or `{status:'ready', screenshot, url}`. Reuse the existing sign-in screenshot UI: display the base64 PNG as an image, map clicks to image pixels, and forward keyboard/scroll events through app actions. Viewing does not itself resume the agent.
-- `input(pauseId, action)` sends the same `SignInAction` click/type/key/scroll commands used by `sendSignInInput`. The CUA accepts human input only while the task is paused; it blocks the helper and main agent from acting at the same time.
-- `completeLogin(pauseId, {allowedHosts, username, password})`, or `completeLogin(pauseId, {otp})`, queues the separate browser-action-only login agent on this computer. **Call only from private server code, never expose this method or credentials as model tool inputs, prompts, progress, logs or job input/results.** Poll `check()` for `login.status: success | needs_input | human_help_needed`, with `input:'otp'` when another code is needed. Helper success leaves the main agent paused.
-- `resume(pauseId)` explicitly resumes; it does not wait for completion. On Continue, call this from the authorized app action, then enqueue the app's continuation job. That job creates `createComputerUseTool({...jobOptions, runKey: originalRunKey, attach:true})`: it polls the same task without sending `/tasks` again. Keep the original saved-session and download options if needed. Stale/duplicate resumes fail with 409; do not silently retry with a new pause ID. If enqueue fails, retry attaching to the already resumed task, not another resume.
+- `requestTakeover(reason, requiredInput?)` requests a pause; `requiredInput` is `human` (default) or `login`. It answers the pause (`taskId`, `pauseId`, `status`, `reason`, `requiredInput`; the same pause again if one exists). Poll `check()` until `waiting_for_input` gives the pause plus the optional login result; while the task is not waiting, `check()` answers its progress (`status`, `steps`, `lastStep`, `summary`, `finalUrl`, `waiting`, `message`).
+- `view(pauseId)` returns `{status:'ready', screenshot, url}`; while the task is still pausing, or once it resumed, the API answers 409 and the call throws. Reuse the existing sign-in screenshot UI: display the base64 PNG as an image, map clicks to image pixels, and forward keyboard/scroll events through app actions. Viewing does not itself resume the agent.
+- `input(pauseId, action)` sends the same `SignInAction` click/type/key/scroll commands used by `sendSignInInput` (`typeText` may be up to 4096 characters here) and answers what the computer did. The CUA accepts human input only while the task is paused; it blocks the helper and main agent from acting at the same time.
+- `completeLogin(pauseId, {allowedHosts, username, password})`, or `completeLogin(pauseId, {otp})`, queues the separate browser-action-only login agent on this computer. **Call only from private server code, never expose this method or credentials as model tool inputs, prompts, progress, logs or job input/results.** Poll `check()` for `login.status: success | needs_input | human_help_needed`, with `input:'otp'` when another code is needed, `reason` (`unverified_success`, `helper_gave_up`, `second_code_requested`, `helper_error:<type>`) when a person is needed, and the helper's short `evidence` (never a private value). Helper success leaves the main agent paused.
+- `resume(pauseId)` explicitly resumes and answers the pause with `status: 'resume_queued'`; it does not wait for completion. On Continue, call this from the authorized app action, then enqueue the app's continuation job. That job creates `createComputerUseTool({...jobOptions, runKey: originalRunKey, attach:true})`: it polls the same task without sending `/tasks` again. Keep the original saved-session and download options if needed. Stale/duplicate resumes fail with 409; do not silently retry with a new pause ID. If enqueue fails, retry attaching to the already resumed task, not another resume.
 - `cancel()` stops the task, including while waiting.
 
 The outer agent may use app-defined tools for takeover and opening the authorized view. Resume requires an explicit user or app decision. Keep any outer-agent conversation in the app's existing job/task storage when ending a waiting job; this SDK does not persist the outer model's messages. The CUA persists its own conversation/resume state under the same task folder as screenshots and progress in its existing result bucket, without passwords or OTPs.

package/dist/computer-use-handoff-tool.d.ts

@@ -41,6 +41,8 @@ export declare function createComputerUseHandoffTools(options: ComputerUseHandof
                 login?: {
                     status: "success" | "needs_input" | "human_help_needed";
                     input?: "otp";
+                    reason?: string;
+                    evidence?: string;
                 };
             }>;
         }) => import("@ai-sdk/provider-utils").ToolResultOutput | PromiseLike<import("@ai-sdk/provider-utils").ToolResultOutput>) | undefined;
@@ -55,6 +57,8 @@ export declare function createComputerUseHandoffTools(options: ComputerUseHandof
             login?: {
                 status: "success" | "needs_input" | "human_help_needed";
                 input?: "otp";
+                reason?: string;
+                evidence?: string;
             };
         }> | undefined;
         execute: import("ai").ToolExecuteFunction<{
@@ -69,6 +73,8 @@ export declare function createComputerUseHandoffTools(options: ComputerUseHandof
             login?: {
                 status: "success" | "needs_input" | "human_help_needed";
                 input?: "otp";
+                reason?: string;
+                evidence?: string;
             };
         }, NoInfer<import("@ai-sdk/provider-utils").Context>>;
     } & {
@@ -101,6 +107,8 @@ export declare function createComputerUseHandoffTools(options: ComputerUseHandof
             login?: {
                 status: "success" | "needs_input" | "human_help_needed";
                 input?: "otp";
+                reason?: string;
+                evidence?: string;
             };
         }, NoInfer<import("@ai-sdk/provider-utils").Context>>;
     }) | ({
@@ -139,6 +147,8 @@ export declare function createComputerUseHandoffTools(options: ComputerUseHandof
                 login?: {
                     status: "success" | "needs_input" | "human_help_needed";
                     input?: "otp";
+                    reason?: string;
+                    evidence?: string;
                 };
             }>;
         }) => import("@ai-sdk/provider-utils").ToolResultOutput | PromiseLike<import("@ai-sdk/provider-utils").ToolResultOutput>) | undefined;
@@ -153,6 +163,8 @@ export declare function createComputerUseHandoffTools(options: ComputerUseHandof
             login?: {
                 status: "success" | "needs_input" | "human_help_needed";
                 input?: "otp";
+                reason?: string;
+                evidence?: string;
             };
         }> | undefined;
         execute: import("ai").ToolExecuteFunction<{
@@ -167,6 +179,8 @@ export declare function createComputerUseHandoffTools(options: ComputerUseHandof
             login?: {
                 status: "success" | "needs_input" | "human_help_needed";
                 input?: "otp";
+                reason?: string;
+                evidence?: string;
             };
         }, NoInfer<import("@ai-sdk/provider-utils").Context>>;
     } & {
@@ -199,6 +213,8 @@ export declare function createComputerUseHandoffTools(options: ComputerUseHandof
             login?: {
                 status: "success" | "needs_input" | "human_help_needed";
                 input?: "otp";
+                reason?: string;
+                evidence?: string;
             };
         }, NoInfer<import("@ai-sdk/provider-utils").Context>>;
     }) | ({
@@ -237,6 +253,8 @@ export declare function createComputerUseHandoffTools(options: ComputerUseHandof
                 login?: {
                     status: "success" | "needs_input" | "human_help_needed";
                     input?: "otp";
+                    reason?: string;
+                    evidence?: string;
                 };
             }>;
         }) => import("@ai-sdk/provider-utils").ToolResultOutput | PromiseLike<import("@ai-sdk/provider-utils").ToolResultOutput>) | undefined;
@@ -251,6 +269,8 @@ export declare function createComputerUseHandoffTools(options: ComputerUseHandof
             login?: {
                 status: "success" | "needs_input" | "human_help_needed";
                 input?: "otp";
+                reason?: string;
+                evidence?: string;
             };
         }> | undefined;
         execute: import("ai").ToolExecuteFunction<{
@@ -265,6 +285,8 @@ export declare function createComputerUseHandoffTools(options: ComputerUseHandof
             login?: {
                 status: "success" | "needs_input" | "human_help_needed";
                 input?: "otp";
+                reason?: string;
+                evidence?: string;
             };
         }, NoInfer<import("@ai-sdk/provider-utils").Context>>;
     } & {
@@ -290,6 +312,8 @@ export declare function createComputerUseHandoffTools(options: ComputerUseHandof
             login?: {
                 status: "success" | "needs_input" | "human_help_needed";
                 input?: "otp";
+                reason?: string;
+                evidence?: string;
             };
         }, NoInfer<import("@ai-sdk/provider-utils").Context>>;
     }) | ({
@@ -328,6 +352,8 @@ export declare function createComputerUseHandoffTools(options: ComputerUseHandof
                 login?: {
                     status: "success" | "needs_input" | "human_help_needed";
                     input?: "otp";
+                    reason?: string;
+                    evidence?: string;
                 };
             }>;
         }) => import("@ai-sdk/provider-utils").ToolResultOutput | PromiseLike<import("@ai-sdk/provider-utils").ToolResultOutput>) | undefined;
@@ -342,6 +368,8 @@ export declare function createComputerUseHandoffTools(options: ComputerUseHandof
             login?: {
                 status: "success" | "needs_input" | "human_help_needed";
                 input?: "otp";
+                reason?: string;
+                evidence?: string;
             };
         }> | undefined;
         execute: import("ai").ToolExecuteFunction<{
@@ -356,6 +384,8 @@ export declare function createComputerUseHandoffTools(options: ComputerUseHandof
             login?: {
                 status: "success" | "needs_input" | "human_help_needed";
                 input?: "otp";
+                reason?: string;
+                evidence?: string;
             };
         }, NoInfer<import("@ai-sdk/provider-utils").Context>>;
     } & {
@@ -381,6 +411,8 @@ export declare function createComputerUseHandoffTools(options: ComputerUseHandof
             login?: {
                 status: "success" | "needs_input" | "human_help_needed";
                 input?: "otp";
+                reason?: string;
+                evidence?: string;
             };
         }, NoInfer<import("@ai-sdk/provider-utils").Context>>;
     });

package/dist/computer-use-handoff.d.ts

@@ -13,9 +13,13 @@ export interface ComputerUseWaiting {
     reason: string;
     requiredInput: 'login' | 'otp' | 'human';
     message?: ComputerUseMessage;
+    /** The private login helper's answer; `reason` says why a person is needed (`unverified_success`, `helper_gave_up`,
+     * `second_code_requested`, `helper_error:<type>`) and `evidence` is the helper's short note on the page, never a private value. */
     login?: {
         status: 'success' | 'needs_input' | 'human_help_needed';
         input?: 'otp';
+        reason?: string;
+        evidence?: string;
     };
 }
 export declare function waitingInput(answer: {
@@ -27,12 +31,34 @@ export declare function waitingInput(answer: {
     message?: unknown;
 }): ComputerUseWaiting;
 export type ComputerUsePage = {
-    status: 'pausing';
-} | {
     status: 'ready';
     screenshot: string;
     url: string;
 };
+/** A pause as the computer-use API describes it on takeover, login and resume; `status` is the task's, not only `waiting_for_input`. */
+export type ComputerUsePause = {
+    taskId: string;
+    pauseId: string;
+    status: string;
+    reason: string;
+    requiredInput: 'login' | 'otp' | 'human';
+};
+/** What the paused computer answered to one input: the desktop function's own result, passed through. */
+export type ComputerUseInputResult = Record<string, unknown>;
+/** Progress of a task that is not waiting: the same fields `createComputerUseTool` reads while it polls. */
+export type ComputerUseProgress = {
+    status: string;
+    steps?: number;
+    lastStep?: string | null;
+    summary?: string | null;
+    finalUrl?: string | null;
+    waiting?: {
+        reason: string;
+        since: string;
+        attempts: number;
+    } | null;
+    message?: ComputerUseMessage;
+};
 export type ComputerUseLoginInput = {
     allowedHosts: readonly string[];
     username: string;
@@ -49,23 +75,14 @@ export interface ComputerUseHandoffOptions extends ComputerUseTransport {
     authorize(): Promise<boolean> | boolean;
 }
 export declare function createComputerUseHandoff(options: ComputerUseHandoffOptions): {
-    requestTakeover(reason: string, requiredInput?: "login" | "human"): Promise<{
-        status: string;
-    }>;
+    requestTakeover(reason: string, requiredInput?: "login" | "human"): Promise<ComputerUsePause>;
     view(pauseId: string): Promise<ComputerUsePage>;
-    input(pauseId: string, action: SignInAction): Promise<{
-        status: "accepted";
-    }>;
+    input(pauseId: string, action: SignInAction): Promise<ComputerUseInputResult>;
     completeLogin(pauseId: string, input: ComputerUseLoginInput): Promise<{
         status: "login_queued";
     }>;
-    check(): Promise<ComputerUseWaiting | {
-        message?: ComputerUseMessage | undefined;
-        status: string;
-    }>;
-    resume(pauseId: string): Promise<{
-        status: "resume_queued";
-    }>;
+    check(): Promise<ComputerUseWaiting | ComputerUseProgress>;
+    resume(pauseId: string): Promise<ComputerUsePause>;
     cancel(): Promise<{
         status: string;
     }>;

package/dist/computer-use-handoff.js

@@ -22,7 +22,11 @@ export function waitingInput(answer) {
         result.message = message;
     const login = answer.login;
     if (login && ['success', 'needs_input', 'human_help_needed'].includes(login.status)) {
-        result.login = { status: login.status, ...(login.input === 'otp' ? { input: 'otp' } : {}) };
+        result.login = {
+            status: login.status, ...(login.input === 'otp' ? { input: 'otp' } : {}),
+            ...(typeof login.reason === 'string' && login.reason ? { reason: login.reason.slice(0, 200) } : {}),
+            ...(typeof login.evidence === 'string' && login.evidence ? { evidence: login.evidence.slice(0, 300) } : {}),
+        };
     }
     return result;
 }
@@ -46,6 +50,11 @@ export function createComputerUseHandoff(options) {
             throw refused(path, answer.status, privateInput ? undefined : answer.body);
         return answer.body;
     };
+    const pauseOf = (answer) => {
+        if (typeof answer.taskId !== 'string' || typeof answer.pauseId !== 'string' || !['login', 'otp', 'human'].includes(String(answer.requiredInput)))
+            throw new Error('The computer use API returned an incomplete pause.');
+        return { taskId: answer.taskId, pauseId: answer.pauseId, status: String(answer.status), reason: typeof answer.reason === 'string' ? answer.reason : 'Outside input needed.', requiredInput: answer.requiredInput };
+    };
     const paused = (pauseId) => {
         if (!pauseId || typeof pauseId !== 'string')
             throw new Error('Computer use handoff needs the current pauseId.');
@@ -55,12 +64,11 @@ export function createComputerUseHandoff(options) {
         requestTakeover(reason, requiredInput = 'human') {
             if (!reason.trim() || !['login', 'human'].includes(requiredInput))
                 throw new Error('Takeover needs a reason and login or human input.');
-            return post('/tasks/takeover', { reason, requiredInput });
+            return post('/tasks/takeover', { reason, requiredInput }).then(pauseOf);
         },
+        // The API answers 409 while the task is still pausing or after it resumed; `ready` is the only 200.
         async view(pauseId) {
             const page = await post('/tasks/view', paused(pauseId));
-            if (page.status === 'pausing')
-                return { status: 'pausing' };
             if (page.status !== 'ready' || typeof page.screenshot !== 'string' || typeof page.url !== 'string')
                 throw new Error('The computer use API returned an incomplete screen.');
             return { status: 'ready', screenshot: page.screenshot, url: page.url };
@@ -88,10 +96,23 @@ export function createComputerUseHandoff(options) {
             if (answer.status === 'waiting_for_input')
                 return waitingInput(answer);
             const message = computerUseMessage(answer.message);
-            return { status: String(answer.status), ...(message ? { message } : {}) };
+            const progress = { status: String(answer.status) };
+            if (typeof answer.steps === 'number')
+                progress.steps = answer.steps;
+            if (answer.last_step !== undefined)
+                progress.lastStep = answer.last_step;
+            if (answer.summary !== undefined)
+                progress.summary = answer.summary;
+            if (answer.final_url !== undefined)
+                progress.finalUrl = answer.final_url;
+            if (answer.waiting !== undefined)
+                progress.waiting = answer.waiting;
+            if (message)
+                progress.message = message;
+            return progress;
         },
         resume(pauseId) {
-            return post('/tasks/resume', paused(pauseId));
+            return post('/tasks/resume', paused(pauseId)).then(pauseOf);
         },
         cancel() {
             return post('/tasks/cancel');

package/dist/computer-use-otp.js

@@ -69,16 +69,15 @@ export async function signInWithOtp(handoff, pauseId, login, otp, options = {})
         const check = await handoff.check();
         if (check.status === 'done' || check.status === 'failed')
             return { status: 'ended', taskStatus: check.status };
-        if (check.status !== 'waiting_for_input') {
-            if (now() >= deadline)
-                return { status: 'human_help_needed', reason: 'The sign-in helper did not finish in time.' };
+        if (now() >= deadline)
+            return { status: 'human_help_needed', reason: 'The sign-in helper did not finish in time.' };
+        if (check.status !== 'waiting_for_input')
             continue;
-        }
         const login = check.login;
         if (login?.status === 'success')
             return { status: 'success' };
         if (login?.status === 'human_help_needed')
-            return { status: 'human_help_needed', reason: 'The sign-in helper could not finish on its own.' };
+            return { status: 'human_help_needed', reason: `The sign-in helper could not finish on its own${login.reason ? ` (${login.reason})` : ''}.` };
         if (login?.status === 'needs_input' && login.input === 'otp') {
             if (answered)
                 return { status: 'human_help_needed', reason: 'The website asked for a second code.' };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@isomorph.ai/app-sdk",
-  "version": "1.41.0",
+  "version": "1.43.0",
   "type": "module",
   "repository": {
     "type": "git",