@isomorph.ai/app-sdk 1.36.3 → 1.37.0

package/README.md

@@ -393,7 +393,7 @@ A job's agent can hand one browser task, writes included, to the computer use se
 import { createComputerUseTool } from '@isomorph.ai/app-sdk/dist/computer-use.js';
 // Inside a job handler:
 const computerUse = createComputerUseTool({ runKey: ctx.runId ?? `my-job/${ctx.scheduledAt}`, progress: ctx.progress, cancelled: ctx.cancelled });
-// Add { computerUse } to generateText's tools. It returns { status: 'done' | 'failed', summary, finalUrl }.
+// Add { computerUse } to generateText's tools. It returns { status: 'done' | 'failed', summary, finalUrl, downloads }.
 ```
 
 - The tool calls the task API at `ISOMORPH_COMPUTER_USE_URL` (region `ISOMORPH_COMPUTER_USE_REGION`, default `us-east-1`); Provision sets both when the cell has the API. Each request is signed with SigV4 (`execute-api`) as the runtime's own role: the Lambda environment's credentials, or on a cluster pod its ServiceAccount role through STS `AssumeRoleWithWebIdentity`.
@@ -409,10 +409,11 @@ Options the app passes, never the model:
 - `session`: the name of a runtime secret a sign-in was saved under (`finishSignIn`'s `saveAs`, below). The task starts signed in and can use only that sign-in's websites. When the task is `done` the refreshed session is saved back to the same secret. A missing one fails with `No saved sign-in named <name>.` Run one task per saved sign-in at a time: two tasks would both save back, and some websites sign out the older copy.
 - `secrets`: `{ name: value }`, at most 10, names `^[A-Za-z][A-Za-z0-9_-]{0,63}$`, values up to 4096 bytes. The computer types a value into a password field by its name; the model sees only the names. Never put a password or any other credential in the task text; pass it here and say "sign in with the `password` secret".
 - `allowedHosts`: the websites (1 to 10 lower-case hostnames) a task with `secrets` and no `session` may use. A saved sign-in carries its own.
+- `onDownload({ name, contentType, size, blob })`: called once for each file a `done` task downloaded, one at a time, before the tool returns (at most 10 files, 100 MiB each). Store the `blob` yourself; the CUA keeps the files for a day at most. Without it nothing is fetched. A link that does not answer 200 fails the tool. The model only sees `downloads: [{ name, size }]`.
 
 A task with `session` or `secrets` works the page through clicks and typing only. `connectionId` is gone: pass `session`.
 
-(1.25.0, 1.36.0; data plane ADR 0078, ADR 0090)
+(1.25.0, 1.36.0, 1.37.0; data plane ADR 0078, ADR 0090, ADR 0091)
 
 ## Browser sign-in for computer use
 

package/dist/computer-use.d.ts

@@ -3,6 +3,12 @@ export interface ComputerUseFile {
     contentType: "application/pdf";
     dataBase64: string;
 }
+export interface ComputerUseDownload {
+    name: string;
+    contentType: string;
+    size: number;
+    blob: Blob;
+}
 export interface ComputerUseOptions {
     /** App-authorized PDFs, at most five files and 2 MiB total. Never supplied by the model. */
     files?: readonly ComputerUseFile[];
@@ -18,6 +24,8 @@ export interface ComputerUseOptions {
      * Never put a credential in the task text.
      */
     secrets?: Readonly<Record<string, string>>;
+    /** The app's code gets each file the task downloaded, one at a time, when the task is done. Never seen by the model. */
+    onDownload?(download: ComputerUseDownload): Promise<void> | void;
     /** The job's runId, or `<job name>/<scheduledAt>` for a scheduled job: one task record per job run. */
     runKey: string;
     /** The job's progress: called on each new step and at least once a minute, which renews the job's lease. */
@@ -35,6 +43,11 @@ export interface ComputerUseResult {
     summary: string | null;
     /** The page the browser was on when the task ended; a sign-in page there means the saved sign-in has expired. */
     finalUrl: string | null;
+    /** The files the task downloaded; their bytes went to onDownload. */
+    downloads: {
+        name: string;
+        size: number;
+    }[];
 }
 export declare function createComputerUseTool(options: ComputerUseOptions): ({
     deferLoading?: boolean;

package/dist/computer-use.js

@@ -2,7 +2,8 @@
 // agent (CUA) and waits for its result. Calls go to the CUA's task API, signed
 // with SigV4 as the runtime's own role (ADR 0078). A saved sign-in is a runtime
 // secret: its session goes with the task and the refreshed one is saved back
-// to it (ADR 0090).
+// to it (ADR 0090). Files the task downloads go to the app's onDownload, never
+// to the model (ADR 0091).
 import { setTimeout as sleep } from 'node:timers/promises';
 import { jsonSchema, tool } from 'ai';
 import { allowedHostsOf, computerUseApi, refused } from './computer-use-api.js';
@@ -71,7 +72,7 @@ export function createComputerUseTool(options) {
         }
     };
     return tool({
-        description: 'Do one task in a web browser, such as filling in and submitting a form on a website. Describe the whole task in plain words, with the website address and any details it needs. Waits until the task ends and returns its status and a summary.',
+        description: 'Do one task in a web browser, such as filling in and submitting a form on a website. Describe the whole task in plain words, with the website address and any details it needs. Waits until the task ends and returns its status and a summary. Files the task downloads go to the app; the result lists their names.',
         inputSchema: jsonSchema({ type: 'object', properties: { task: { type: 'string', minLength: 1 } }, required: ['task'], additionalProperties: false }),
         execute: async ({ task }) => {
             await api.post('/tasks', {
@@ -101,7 +102,17 @@ export function createComputerUseTool(options) {
                     if (answer.status === 'done' || answer.status === 'failed') {
                         if (answer.status === 'done' && session !== undefined && answer.session)
                             await runtimeSecrets.put(session, JSON.stringify(answer.session), transport);
-                        return { status: answer.status, summary: answer.summary ?? null, finalUrl: answer.final_url ?? null };
+                        const downloads = answer.downloads ?? [];
+                        if (options.onDownload) {
+                            for (const { name, contentType, size, url } of downloads) {
+                                await options.progress(99, `Computer use download: ${name}`);
+                                const response = await (options.fetch ?? fetch)(url);
+                                if (response.status !== 200)
+                                    throw refused(`the download ${name}`, response.status, undefined);
+                                await options.onDownload({ name, contentType, size, blob: await response.blob() });
+                            }
+                        }
+                        return { status: answer.status, summary: answer.summary ?? null, finalUrl: answer.final_url ?? null, downloads: downloads.map(({ name, size }) => ({ name, size })) };
                     }
                     if (answer.steps > steps || now() - reported >= HEARTBEAT_MS) {
                         steps = Math.max(steps, answer.steps);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@isomorph.ai/app-sdk",
-  "version": "1.36.3",
+  "version": "1.37.0",
   "type": "module",
   "repository": {
     "type": "git",