@isomorph.ai/app-sdk 1.24.0 → 1.26.0

package/README.md

@@ -370,6 +370,25 @@ try {
 
 App code declares exact HTTPS pages and named click/fill selectors reviewed for reads. Tools provide navigate, observe, control, scroll, screenshot and close. Observations are untrusted webpage content. One isolated page per run, one active run per process, 50 actions, five minutes, ten-second operations, bounded text/screenshots. Private destinations, undeclared navigation, downloads, popups, WebSockets, service workers and non-GET/HEAD requests are blocked. GET-only filtering cannot infer website business semantics: never declare URLs/controls that perform external writes. Chromium's sandbox remains enabled; unsupported sandbox environments fail clearly.
 
+## Computer use tool
+
+A job's agent can hand one browser task, writes included, to Harbour's computer use agent. Use it only in `jobs/*.ts`. Install the optional peer `ai`.
+
+```ts
+import { createComputerUseTool } from '@isomorph.ai/app-sdk/dist/computer-use.js';
+// Inside a job handler:
+const computerUse = createComputerUseTool({ runKey: ctx.runId ?? `my-job/${ctx.scheduledAt}`, progress: ctx.progress, cancelled: ctx.cancelled });
+// Add { computerUse } to generateText's tools. It returns { status: 'done' | 'failed', summary }.
+```
+
+- The tool calls the task API at `ISOMORPH_COMPUTER_USE_URL` (region `ISOMORPH_COMPUTER_USE_REGION`, default `us-east-1`); Provision sets both when the cell has the API. Each request is signed with SigV4 (`execute-api`) as the runtime's own role: the Lambda environment's credentials, or on a cluster pod its ServiceAccount role through STS `AssumeRoleWithWebIdentity`.
+- It starts the task with `POST /tasks`, then checks `POST /tasks/check` every 5 s until the task is `done` or `failed`.
+- It calls `progress(step count, note)` on every new step and at least once a minute, so the job's lease does not run out.
+- When `cancelled()` turns true it sends `POST /tasks/cancel` once and keeps checking until the task has `failed`.
+- One task record per `runKey`: a retried job run sends the same key and gets the queued, running or finished task back. Only a failed task starts a new attempt.
+
+(1.25.0; data plane ADR 0078)
+
 ## Running and rollout
 
 `isomorph dev` uses canned server-model fixtures. `dev --real` supplies an approved development connection to the action process. Hosted actions resolve their exact ACTIVE version key. The current V2 PLAN is published after LIVE; before key activation, an AI action fails and later actions retry. LIVE alone is not AI acceptance.

package/dist/aws.d.ts

@@ -16,3 +16,23 @@ export declare function sendQueueMessage(aws: AwsEnvironment, queueUrl: string,
     fetch?: typeof fetch;
     signingDate?: Date;
 }): Promise<void>;
+type Credentials = Omit<AwsEnvironment, 'region'>;
+/**
+ * The runtime role's credentials: the Lambda environment's, or on a cluster
+ * pod the ServiceAccount role's (IRSA: AWS_ROLE_ARN and the projected token
+ * in AWS_WEB_IDENTITY_TOKEN_FILE), exchanged with STS
+ * AssumeRoleWithWebIdentity and kept until five minutes before they expire.
+ */
+export declare function runtimeCredentials(env?: NodeJS.ProcessEnv, { fetch: fetcher, now }?: {
+    fetch?: typeof fetch;
+    now?: () => number;
+}): Promise<Credentials>;
+/** A JSON POST signed for `service` in `aws.region`, answered with its status and JSON body. */
+export declare function signedJsonPost(aws: AwsEnvironment, service: string, target: string, payload: unknown, { fetch: fetcher, signingDate }?: {
+    fetch?: typeof fetch;
+    signingDate?: Date;
+}): Promise<{
+    status: number;
+    body: unknown;
+}>;
+export {};

package/dist/aws.js

@@ -1,6 +1,8 @@
-// The AWS calls a serverless runtime makes with its function role, signed
-// here rather than through an AWS SDK client (ADR 0065). Credentials and the
-// region come from the Lambda environment on every call.
+// The AWS calls a runtime makes with its own role, signed here rather than
+// through an AWS SDK client (ADR 0065). On a serverless cell the credentials
+// and the region come from the Lambda environment on every call; on a cluster
+// pod, from the ServiceAccount's role (runtimeCredentials).
+import { readFile } from 'node:fs/promises';
 import { Sha256 } from '@aws-crypto/sha256-js';
 import { HttpRequest } from '@smithy/protocol-http';
 import { SignatureV4 } from '@smithy/signature-v4';
@@ -49,3 +51,43 @@ export async function sendQueueMessage(aws, queueUrl, body, { fetch: fetcher = f
         throw new Error(`SQS refused the message (${response.status}${answer?.__type ? ` ${answer.__type}` : ''}${answer?.message ? `: ${answer.message}` : ''})`);
     }
 }
+let webIdentity;
+/**
+ * The runtime role's credentials: the Lambda environment's, or on a cluster
+ * pod the ServiceAccount role's (IRSA: AWS_ROLE_ARN and the projected token
+ * in AWS_WEB_IDENTITY_TOKEN_FILE), exchanged with STS
+ * AssumeRoleWithWebIdentity and kept until five minutes before they expire.
+ */
+export async function runtimeCredentials(env = process.env, { fetch: fetcher = fetch, now = Date.now } = {}) {
+    if (env.AWS_ACCESS_KEY_ID && env.AWS_SECRET_ACCESS_KEY)
+        return { accessKeyId: env.AWS_ACCESS_KEY_ID, secretAccessKey: env.AWS_SECRET_ACCESS_KEY, ...(env.AWS_SESSION_TOKEN ? { sessionToken: env.AWS_SESSION_TOKEN } : {}) };
+    const roleArn = env.AWS_ROLE_ARN;
+    const tokenFile = env.AWS_WEB_IDENTITY_TOKEN_FILE;
+    if (!roleArn || !tokenFile)
+        throw new Error('the runtime has no AWS credentials (AWS_ACCESS_KEY_ID, or AWS_ROLE_ARN with AWS_WEB_IDENTITY_TOKEN_FILE)');
+    if (webIdentity?.roleArn === roleArn && webIdentity.expiresAt - now() > 5 * 60_000)
+        return webIdentity.credentials;
+    const region = env.AWS_REGION || env.AWS_DEFAULT_REGION || 'us-east-1';
+    // The token is read on each exchange: the kubelet rotates the file.
+    const body = new URLSearchParams({ Action: 'AssumeRoleWithWebIdentity', Version: '2011-06-15', RoleArn: roleArn, RoleSessionName: env.AWS_ROLE_SESSION_NAME || 'isomorph-runtime', WebIdentityToken: (await readFile(tokenFile, 'utf8')).trim() });
+    const response = await fetcher(`https://sts.${region}.amazonaws.com/`, { method: 'POST', headers: { 'content-type': 'application/x-www-form-urlencoded' }, body: body.toString(), signal: AbortSignal.timeout(10_000) });
+    const answer = await response.text();
+    const field = (name) => answer.match(new RegExp(`<${name}>([^<]+)</${name}>`))?.[1];
+    const accessKeyId = field('AccessKeyId'), secretAccessKey = field('SecretAccessKey'), sessionToken = field('SessionToken'), expiration = Date.parse(field('Expiration') ?? '');
+    if (!response.ok || !accessKeyId || !secretAccessKey || !sessionToken || Number.isNaN(expiration))
+        throw new Error(`STS did not issue the runtime role's credentials (${response.status}${field('Code') ? ` ${field('Code')}` : ''})`);
+    webIdentity = { roleArn, credentials: { accessKeyId, secretAccessKey, sessionToken }, expiresAt: expiration };
+    return webIdentity.credentials;
+}
+/** A JSON POST signed for `service` in `aws.region`, answered with its status and JSON body. */
+export async function signedJsonPost(aws, service, target, payload, { fetch: fetcher = fetch, signingDate = new Date() } = {}) {
+    const url = new URL(target);
+    const body = JSON.stringify(payload);
+    const request = new HttpRequest({
+        method: 'POST', protocol: url.protocol, hostname: url.hostname, ...(url.port ? { port: Number(url.port) } : {}), path: url.pathname,
+        headers: { host: url.host, 'content-type': 'application/json' }, body,
+    });
+    const { host: _host, ...headers } = (await signer(aws, service).sign(request, { signingDate })).headers;
+    const response = await fetcher(url.href, { method: 'POST', headers, body, signal: AbortSignal.timeout(30_000) });
+    return { status: response.status, body: await response.json().catch(() => undefined) };
+}

package/dist/computer-use.d.ts

@@ -0,0 +1,227 @@
+export interface ComputerUseFile {
+    name: string;
+    contentType: "application/pdf";
+    dataBase64: string;
+}
+export interface ComputerUseOptions {
+    /** App-authorized PDFs, at most five files and 2 MiB total. Never supplied by the model. */
+    files?: readonly ComputerUseFile[];
+    /** The job's runId, or `<job name>/<scheduledAt>` for a scheduled job: one task record per job run. */
+    runKey: string;
+    /** The job's progress: called on each new step and at least once a minute, which renews the job's lease. */
+    progress(percent: number, note?: string): Promise<void> | void;
+    /** The job's cancelled(): when true the tool cancels the task and waits until the CUA stops. */
+    cancelled(): Promise<boolean> | boolean;
+    env?: NodeJS.ProcessEnv;
+    fetch?: typeof fetch;
+    /** The pause between checks; the tests shorten it. */
+    pollIntervalMs?: number;
+    now?: () => number;
+}
+export interface ComputerUseResult {
+    status: 'done' | 'failed';
+    summary: string | null;
+}
+export declare function createComputerUseTool(options: ComputerUseOptions): ({
+    deferLoading?: boolean;
+    title?: string;
+    providerOptions?: import("@ai-sdk/provider-utils").ProviderOptions;
+    metadata?: import("@ai-sdk/provider").JSONObject;
+    inputSchema: import("ai").FlexibleSchema<{
+        task: string;
+    }>;
+    contextSchema?: import("ai").FlexibleSchema<import("@ai-sdk/provider-utils").Context> | undefined;
+    needsApproval?: boolean | import("@ai-sdk/provider-utils").ToolNeedsApprovalFunction<{
+        task: string;
+    }, NoInfer<import("@ai-sdk/provider-utils").Context>> | undefined;
+    onInputStart?: ((options: import("ai").ToolExecutionOptions<NoInfer<import("@ai-sdk/provider-utils").Context>>) => void | PromiseLike<void>) | undefined;
+    onInputDelta?: ((options: {
+        inputTextDelta: string;
+    } & import("ai").ToolExecutionOptions<NoInfer<import("@ai-sdk/provider-utils").Context>>) => void | PromiseLike<void>) | undefined;
+    onInputAvailable?: ((options: {
+        input: {
+            task: string;
+        };
+    } & import("ai").ToolExecutionOptions<NoInfer<import("@ai-sdk/provider-utils").Context>>) => void | PromiseLike<void>) | undefined;
+    toModelOutput?: ((options: {
+        toolCallId: string;
+        input: {
+            task: string;
+        };
+        output: NoInfer<ComputerUseResult>;
+    }) => import("@ai-sdk/provider-utils").ToolResultOutput | PromiseLike<import("@ai-sdk/provider-utils").ToolResultOutput>) | undefined;
+} & {
+    outputSchema?: import("ai").FlexibleSchema<ComputerUseResult> | undefined;
+    execute: import("ai").ToolExecuteFunction<{
+        task: string;
+    }, ComputerUseResult, NoInfer<import("@ai-sdk/provider-utils").Context>>;
+} & {
+    description?: string | ((options: {
+        context: NoInfer<import("@ai-sdk/provider-utils").Context>;
+        experimental_sandbox?: import("ai").Experimental_SandboxSession;
+    }) => string) | undefined;
+    strict?: boolean;
+    inputExamples?: {
+        input: NoInfer<{
+            task: string;
+        }>;
+    }[] | undefined;
+    id?: never;
+    isProviderExecuted?: never;
+    args?: never;
+    supportsDeferredResults?: never;
+} & {
+    type?: undefined | "function";
+} & {
+    execute: import("ai").ToolExecuteFunction<{
+        task: string;
+    }, ComputerUseResult, NoInfer<import("@ai-sdk/provider-utils").Context>>;
+}) | ({
+    deferLoading?: boolean;
+    title?: string;
+    providerOptions?: import("@ai-sdk/provider-utils").ProviderOptions;
+    metadata?: import("@ai-sdk/provider").JSONObject;
+    inputSchema: import("ai").FlexibleSchema<{
+        task: string;
+    }>;
+    contextSchema?: import("ai").FlexibleSchema<import("@ai-sdk/provider-utils").Context> | undefined;
+    needsApproval?: boolean | import("@ai-sdk/provider-utils").ToolNeedsApprovalFunction<{
+        task: string;
+    }, NoInfer<import("@ai-sdk/provider-utils").Context>> | undefined;
+    onInputStart?: ((options: import("ai").ToolExecutionOptions<NoInfer<import("@ai-sdk/provider-utils").Context>>) => void | PromiseLike<void>) | undefined;
+    onInputDelta?: ((options: {
+        inputTextDelta: string;
+    } & import("ai").ToolExecutionOptions<NoInfer<import("@ai-sdk/provider-utils").Context>>) => void | PromiseLike<void>) | undefined;
+    onInputAvailable?: ((options: {
+        input: {
+            task: string;
+        };
+    } & import("ai").ToolExecutionOptions<NoInfer<import("@ai-sdk/provider-utils").Context>>) => void | PromiseLike<void>) | undefined;
+    toModelOutput?: ((options: {
+        toolCallId: string;
+        input: {
+            task: string;
+        };
+        output: NoInfer<ComputerUseResult>;
+    }) => import("@ai-sdk/provider-utils").ToolResultOutput | PromiseLike<import("@ai-sdk/provider-utils").ToolResultOutput>) | undefined;
+} & {
+    outputSchema?: import("ai").FlexibleSchema<ComputerUseResult> | undefined;
+    execute: import("ai").ToolExecuteFunction<{
+        task: string;
+    }, ComputerUseResult, NoInfer<import("@ai-sdk/provider-utils").Context>>;
+} & {
+    description?: string | ((options: {
+        context: NoInfer<import("@ai-sdk/provider-utils").Context>;
+        experimental_sandbox?: import("ai").Experimental_SandboxSession;
+    }) => string) | undefined;
+    strict?: boolean;
+    inputExamples?: {
+        input: NoInfer<{
+            task: string;
+        }>;
+    }[] | undefined;
+    id?: never;
+    isProviderExecuted?: never;
+    args?: never;
+    supportsDeferredResults?: never;
+} & {
+    type: "dynamic";
+} & {
+    execute: import("ai").ToolExecuteFunction<{
+        task: string;
+    }, ComputerUseResult, NoInfer<import("@ai-sdk/provider-utils").Context>>;
+}) | ({
+    deferLoading?: boolean;
+    title?: string;
+    providerOptions?: import("@ai-sdk/provider-utils").ProviderOptions;
+    metadata?: import("@ai-sdk/provider").JSONObject;
+    inputSchema: import("ai").FlexibleSchema<{
+        task: string;
+    }>;
+    contextSchema?: import("ai").FlexibleSchema<import("@ai-sdk/provider-utils").Context> | undefined;
+    needsApproval?: boolean | import("@ai-sdk/provider-utils").ToolNeedsApprovalFunction<{
+        task: string;
+    }, NoInfer<import("@ai-sdk/provider-utils").Context>> | undefined;
+    onInputStart?: ((options: import("ai").ToolExecutionOptions<NoInfer<import("@ai-sdk/provider-utils").Context>>) => void | PromiseLike<void>) | undefined;
+    onInputDelta?: ((options: {
+        inputTextDelta: string;
+    } & import("ai").ToolExecutionOptions<NoInfer<import("@ai-sdk/provider-utils").Context>>) => void | PromiseLike<void>) | undefined;
+    onInputAvailable?: ((options: {
+        input: {
+            task: string;
+        };
+    } & import("ai").ToolExecutionOptions<NoInfer<import("@ai-sdk/provider-utils").Context>>) => void | PromiseLike<void>) | undefined;
+    toModelOutput?: ((options: {
+        toolCallId: string;
+        input: {
+            task: string;
+        };
+        output: NoInfer<ComputerUseResult>;
+    }) => import("@ai-sdk/provider-utils").ToolResultOutput | PromiseLike<import("@ai-sdk/provider-utils").ToolResultOutput>) | undefined;
+} & {
+    outputSchema?: import("ai").FlexibleSchema<ComputerUseResult> | undefined;
+    execute: import("ai").ToolExecuteFunction<{
+        task: string;
+    }, ComputerUseResult, NoInfer<import("@ai-sdk/provider-utils").Context>>;
+} & {
+    type: "provider";
+    id: `${string}.${string}`;
+    args: Record<string, unknown>;
+    description?: never;
+    strict?: never;
+    inputExamples?: never;
+} & {
+    isProviderExecuted: false;
+    supportsDeferredResults?: never;
+} & {
+    execute: import("ai").ToolExecuteFunction<{
+        task: string;
+    }, ComputerUseResult, NoInfer<import("@ai-sdk/provider-utils").Context>>;
+}) | ({
+    deferLoading?: boolean;
+    title?: string;
+    providerOptions?: import("@ai-sdk/provider-utils").ProviderOptions;
+    metadata?: import("@ai-sdk/provider").JSONObject;
+    inputSchema: import("ai").FlexibleSchema<{
+        task: string;
+    }>;
+    contextSchema?: import("ai").FlexibleSchema<import("@ai-sdk/provider-utils").Context> | undefined;
+    needsApproval?: boolean | import("@ai-sdk/provider-utils").ToolNeedsApprovalFunction<{
+        task: string;
+    }, NoInfer<import("@ai-sdk/provider-utils").Context>> | undefined;
+    onInputStart?: ((options: import("ai").ToolExecutionOptions<NoInfer<import("@ai-sdk/provider-utils").Context>>) => void | PromiseLike<void>) | undefined;
+    onInputDelta?: ((options: {
+        inputTextDelta: string;
+    } & import("ai").ToolExecutionOptions<NoInfer<import("@ai-sdk/provider-utils").Context>>) => void | PromiseLike<void>) | undefined;
+    onInputAvailable?: ((options: {
+        input: {
+            task: string;
+        };
+    } & import("ai").ToolExecutionOptions<NoInfer<import("@ai-sdk/provider-utils").Context>>) => void | PromiseLike<void>) | undefined;
+    toModelOutput?: ((options: {
+        toolCallId: string;
+        input: {
+            task: string;
+        };
+        output: NoInfer<ComputerUseResult>;
+    }) => import("@ai-sdk/provider-utils").ToolResultOutput | PromiseLike<import("@ai-sdk/provider-utils").ToolResultOutput>) | undefined;
+} & {
+    outputSchema?: import("ai").FlexibleSchema<ComputerUseResult> | undefined;
+    execute: import("ai").ToolExecuteFunction<{
+        task: string;
+    }, ComputerUseResult, NoInfer<import("@ai-sdk/provider-utils").Context>>;
+} & {
+    type: "provider";
+    id: `${string}.${string}`;
+    args: Record<string, unknown>;
+    description?: never;
+    strict?: never;
+    inputExamples?: never;
+} & {
+    isProviderExecuted: true;
+    supportsDeferredResults?: boolean;
+} & {
+    execute: import("ai").ToolExecuteFunction<{
+        task: string;
+    }, ComputerUseResult, NoInfer<import("@ai-sdk/provider-utils").Context>>;
+});

package/dist/computer-use.js

@@ -0,0 +1,90 @@
+// The computer use tool: hands one browser task to Harbour's computer use
+// agent (CUA) and waits for its result. Calls go to the CUA's task API, signed
+// with SigV4 as the runtime's own role (ADR 0078).
+import { setTimeout as sleep } from 'node:timers/promises';
+import { jsonSchema, tool } from 'ai';
+import { runtimeCredentials, signedJsonPost } from './aws.js';
+const HEARTBEAT_MS = 60_000;
+// Checks the API did not answer (a throttle, a 5xx, no connection) before the tool gives up: about a minute.
+const MAX_MISSED_CHECKS = 12;
+export function createComputerUseTool(options) {
+    const files = options.files?.map(file => ({ ...file }));
+    if (files) {
+        const names = new Set();
+        let total = 0;
+        if (files.length > 5)
+            throw new Error("Computer use accepts at most five PDFs.");
+        for (const file of files) {
+            if (!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,119}\.pdf$/.test(file.name) || names.has(file.name))
+                throw new Error("Computer use PDF names must be unique safe basenames.");
+            if (file.contentType !== "application/pdf" || typeof file.dataBase64 !== "string" || file.dataBase64.length > Math.ceil(2 * 1024 * 1024 / 3) * 4)
+                throw new Error("Computer use needs base64 PDFs, at most 2 MiB total.");
+            const bytes = Buffer.from(file.dataBase64, "base64");
+            total += bytes.length;
+            if (bytes.toString("base64") !== file.dataBase64 || bytes.subarray(0, 5).toString() !== "%PDF-" || total > 2 * 1024 * 1024)
+                throw new Error("Computer use needs PDFs, at most 2 MiB total.");
+            names.add(file.name);
+        }
+    }
+    const env = options.env ?? process.env;
+    const base = env.ISOMORPH_COMPUTER_USE_URL?.replace(/\/+$/, '');
+    if (!base)
+        throw new Error('This app has no computer use API (ISOMORPH_COMPUTER_USE_URL is not set).');
+    if (!options.runKey)
+        throw new Error('createComputerUseTool needs the job run as runKey.');
+    const region = env.ISOMORPH_COMPUTER_USE_REGION || 'us-east-1';
+    const now = options.now ?? Date.now;
+    const transport = options.fetch ? { fetch: options.fetch } : {};
+    const call = async (path, body) => signedJsonPost({ ...await runtimeCredentials(env, transport), region }, 'execute-api', `${base}${path}`, body, transport);
+    const refused = (path, status, body) => {
+        const error = body?.error;
+        return new Error(`The computer use API refused ${path} (${status}${typeof error === 'string' ? `: ${error}` : ''}).`);
+    };
+    return tool({
+        description: 'Do one task in a web browser, such as filling in and submitting a form on a website. Describe the whole task in plain words, with the website address and any details it needs. Waits until the task ends and returns its status and a summary.',
+        inputSchema: jsonSchema({ type: 'object', properties: { task: { type: 'string', minLength: 1 } }, required: ['task'], additionalProperties: false }),
+        execute: async ({ task }) => {
+            const started = await call('/tasks', { run_key: options.runKey, task, ...(files?.length ? { files } : {}) });
+            if (started.status !== 200)
+                throw refused('/tasks', started.status, started.body);
+            await options.progress(0, 'Computer use task started.');
+            let reported = now();
+            let steps = 0;
+            let cancelSent = false;
+            let missed = 0;
+            for (;;) {
+                if (!cancelSent && await options.cancelled()) {
+                    const cancel = await call('/tasks/cancel', { run_key: options.runKey });
+                    if (cancel.status !== 200)
+                        throw refused('/tasks/cancel', cancel.status, cancel.body);
+                    cancelSent = true;
+                }
+                let check;
+                try {
+                    check = await call('/tasks/check', { run_key: options.runKey });
+                }
+                catch {
+                    check = undefined;
+                }
+                if (check?.status === 200) {
+                    missed = 0;
+                    const answer = check.body;
+                    if (answer.status === 'done' || answer.status === 'failed')
+                        return { status: answer.status, summary: answer.summary ?? null };
+                    if (answer.steps > steps || now() - reported >= HEARTBEAT_MS) {
+                        steps = Math.max(steps, answer.steps);
+                        await options.progress(Math.min(steps, 99), answer.last_step ? `Computer use step ${steps}: ${answer.last_step}` : `Computer use task ${answer.status}.`);
+                        reported = now();
+                    }
+                }
+                else if (check && check.status !== 429 && check.status < 500) {
+                    throw refused('/tasks/check', check.status, check.body);
+                }
+                else if (++missed > MAX_MISSED_CHECKS) {
+                    throw new Error('The computer use API stopped answering.');
+                }
+                await sleep(options.pollIntervalMs ?? 5_000);
+            }
+        },
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@isomorph.ai/app-sdk",
-  "version": "1.24.0",
+  "version": "1.26.0",
   "type": "module",
   "repository": {
     "type": "git",