@shardflux/sdk 0.14.0 → 0.15.0

package/dist/computer.d.ts

@@ -0,0 +1,143 @@
+/**
+ * Computer use (0.15.0+, contracts §45): the workspace desktop, which the platform starts on the first call that needs
+ * it. `workspace.computer` drives it; `computerToolset(workspace)` answers Claude's computer toolset
+ * (`computer_toolset_20260801`) with one batch per model turn.
+ *
+ *   await workspace.setComputerUse(true);                  // or open({ ..., computerUse: true })
+ *   const shot = await workspace.computer.screenshot();     // { format: 'png', width: 1280, height: 800, data }
+ *   await workspace.computer.act([{ action: 'left_click', coordinate: [640, 400] }, { action: 'type', text: 'hello' }]);
+ *   const { url } = await workspace.computer.stream();      // a private link to watch the screen
+ */
+import type { CellClient, ComputerAction, ComputerActionsResult, ComputerImage, ComputerStatus } from './cell.js';
+import type { ComputerUse } from './client.js';
+import type { WorkspacePorts } from './ports.js';
+/** A decoded screen image. */
+export interface ComputerScreenshot {
+    format: 'png' | 'jpeg';
+    width: number;
+    height: number;
+    data: Uint8Array;
+}
+export interface ComputerActOptions {
+    /** Append a screenshot after the last action that ran (also after a failure). */
+    screenshot?: boolean;
+    /** Wait this long before that screenshot when an action changed the screen (default 250). */
+    settleMs?: number;
+    format?: 'png' | 'jpeg';
+    /** JPEG quality 1..100 (default 80). */
+    quality?: number;
+    signal?: AbortSignal;
+}
+export interface ComputerStreamOptions {
+    /** Let the viewer use the mouse and keyboard (default false: view only, enforced in the workspace). */
+    interactive?: boolean;
+    /** How long the link works (60..604800 s; default 86400). */
+    ttlSeconds?: number;
+}
+/** A private link to the desktop's viewer: open it in a browser or an iframe. */
+export interface ComputerStream {
+    url: string;
+    expiresAt: string;
+    port: number;
+    interactive: boolean;
+}
+/** Decodes a result image (base64 in the API) to bytes. */
+export declare function decodeComputerImage(img: ComputerImage): ComputerScreenshot;
+interface ComputerDeps {
+    cell: () => CellClient;
+    ports: () => WorkspacePorts;
+    setEnabled: (enabled: boolean | null) => Promise<ComputerUse>;
+}
+/** The workspace desktop (workspace.computer). */
+export declare class WorkspaceComputer {
+    #private;
+    constructor(deps: ComputerDeps);
+    /**
+     * Runs actions in order (the first failure stops the batch; later actions come back `skipped`), optionally ending
+     * with a screenshot. The actions are Claude's computer toolset members with their parameter names.
+     */
+    act(actions: ComputerAction[], opts?: ComputerActOptions): Promise<ComputerActionsResult>;
+    /** The screen now (the pointer drawn in). */
+    screenshot(opts?: Pick<ComputerActOptions, 'format' | 'quality' | 'signal'>): Promise<ComputerScreenshot>;
+    /** Whether the desktop runs, its size and its viewers. Never starts it. */
+    status(): Promise<ComputerStatus>;
+    /** Starts the desktop (a no-op while it runs; the size applies to a start only: 640x480..2560x1600, default 1280x800). */
+    start(size?: {
+        width?: number;
+        height?: number;
+    }): Promise<ComputerStatus>;
+    /** Stops the desktop; its windows close. The next call that needs it starts a fresh one. */
+    stop(): Promise<void>;
+    /**
+     * A private link to watch the desktop (or use it, with `interactive`): starts the viewer in the workspace, exposes
+     * its port and signs a link to the viewer page (inbound ports, contracts §39). Anyone with the link can open it until
+     * it expires; `stopStream()` ends every view.
+     */
+    stream(opts?: ComputerStreamOptions): Promise<ComputerStream>;
+    /** Ends every view and closes the viewer ports (their links stop working). */
+    stopStream(): Promise<void>;
+    /** Switches computer use on or off for this workspace (null follows the template). */
+    setEnabled(enabled: boolean | null): Promise<ComputerUse>;
+}
+/** The `tools` entry of Claude's computer toolset (no name, no display size). */
+export declare const COMPUTER_TOOLSET: {
+    readonly type: "computer_toolset_20260801";
+};
+/** A `tool_use` content block (structurally the Anthropic SDK's, so its blocks pass as they are). */
+export interface ToolUseBlockLike {
+    type: string;
+    id: string;
+    name: string;
+    input: unknown;
+    toolset_name?: string | null;
+}
+type ResultContent = Array<{
+    type: 'text';
+    text: string;
+} | {
+    type: 'image';
+    source: {
+        type: 'base64';
+        media_type: 'image/png' | 'image/jpeg';
+        data: string;
+    };
+}>;
+/** A `tool_result` block answering a toolset call (each echoes `toolset_name: "computer"`, as the API requires). */
+export interface ComputerToolResult {
+    type: 'tool_result';
+    tool_use_id: string;
+    toolset_name: 'computer';
+    content: ResultContent;
+    is_error?: true;
+}
+export interface ComputerToolsetOptions {
+    /** Screenshot format (default png). */
+    format?: 'png' | 'jpeg';
+    quality?: number;
+    settleMs?: number;
+    signal?: AbortSignal;
+}
+/**
+ * Answers Claude's computer toolset (`tools: [COMPUTER_TOOLSET]`, Claude Opus 5.5 / Sonnet 5.5 and later on the
+ * Claude API) from this workspace's desktop:
+ *
+ *   const computer = computerToolset(workspace);
+ *   const msg = await anthropic.messages.create({ model, max_tokens, tools: [computer.definition], messages });
+ *   messages.push({ role: 'assistant', content: msg.content });
+ *   messages.push({ role: 'user', content: await computer.run(msg.content) });
+ *
+ * `run` takes the response content, runs every `toolset_name: "computer"` call of the turn as one batch (in order;
+ * the first failure stops it and the rest are answered "Not executed"), and returns one `tool_result` per call:
+ * images for screenshot and zoom, the position for cursor_position, `OK` otherwise. When the batch does not end with a
+ * look, a screenshot is attached to the last result, so the model sees the outcome without another round trip. A
+ * batch the desktop refuses as invalid (a coordinate outside the screen) is answered with the reason on every call.
+ */
+export declare function computerToolset(workspace: {
+    computer: WorkspaceComputer;
+}): {
+    definition: {
+        readonly type: "computer_toolset_20260801";
+    };
+    run(content: readonly unknown[], opts?: ComputerToolsetOptions): Promise<ComputerToolResult[]>;
+};
+export {};

package/dist/computer.js

@@ -0,0 +1,146 @@
+import { ShardfluxApiError, ShardfluxProtocolError } from "./errors.js";
+/** Decodes a result image (base64 in the API) to bytes. */
+export function decodeComputerImage(img) {
+    return { format: img.format, width: img.width, height: img.height, data: new Uint8Array(Buffer.from(img.data, 'base64')) };
+}
+/** The workspace desktop (workspace.computer). */
+export class WorkspaceComputer {
+    #deps;
+    constructor(deps) {
+        this.#deps = deps;
+    }
+    /**
+     * Runs actions in order (the first failure stops the batch; later actions come back `skipped`), optionally ending
+     * with a screenshot. The actions are Claude's computer toolset members with their parameter names.
+     */
+    act(actions, opts = {}) {
+        return this.#deps.cell().computer.act({
+            actions,
+            ...(opts.screenshot !== undefined ? { screenshot: opts.screenshot } : {}),
+            ...(opts.settleMs !== undefined ? { settle_ms: opts.settleMs } : {}),
+            ...(opts.format !== undefined ? { format: opts.format } : {}),
+            ...(opts.quality !== undefined ? { quality: opts.quality } : {}),
+        }, opts.signal);
+    }
+    /** The screen now (the pointer drawn in). */
+    async screenshot(opts = {}) {
+        const r = await this.act([], { ...opts, screenshot: true });
+        if (!r.screenshot)
+            throw new ShardfluxProtocolError('POST /computer/actions: the desktop returned no screenshot', 200, 'cell');
+        return decodeComputerImage(r.screenshot);
+    }
+    /** Whether the desktop runs, its size and its viewers. Never starts it. */
+    status() {
+        return this.#deps.cell().computer.status();
+    }
+    /** Starts the desktop (a no-op while it runs; the size applies to a start only: 640x480..2560x1600, default 1280x800). */
+    start(size = {}) {
+        return this.#deps.cell().computer.start(size);
+    }
+    /** Stops the desktop; its windows close. The next call that needs it starts a fresh one. */
+    stop() {
+        return this.#deps.cell().computer.stop();
+    }
+    /**
+     * A private link to watch the desktop (or use it, with `interactive`): starts the viewer in the workspace, exposes
+     * its port and signs a link to the viewer page (inbound ports, contracts §39). Anyone with the link can open it until
+     * it expires; `stopStream()` ends every view.
+     */
+    async stream(opts = {}) {
+        const info = await this.#deps.cell().computer.streamStart({ interactive: opts.interactive ?? false });
+        const ports = this.#deps.ports();
+        await ports.expose(info.port);
+        const link = await ports.link(info.port, { path: info.path, ...(opts.ttlSeconds !== undefined ? { ttlSeconds: opts.ttlSeconds } : {}) });
+        return { url: link.url, expiresAt: link.expiresAt, port: info.port, interactive: info.interactive };
+    }
+    /** Ends every view and closes the viewer ports (their links stop working). */
+    async stopStream() {
+        const ports = this.#deps.ports();
+        await Promise.all([ports.close(61002), ports.close(61003)]);
+        await this.#deps.cell().computer.streamStop();
+    }
+    /** Switches computer use on or off for this workspace (null follows the template). */
+    setEnabled(enabled) {
+        return this.#deps.setEnabled(enabled);
+    }
+}
+// ---- Claude's computer toolset ------------------------------------------------------------------------------------------
+/** The `tools` entry of Claude's computer toolset (no name, no display size). */
+export const COMPUTER_TOOLSET = { type: 'computer_toolset_20260801' };
+/** Parameters the toolset members take; anything else a model sends is not forwarded. */
+const PARAMS = ['coordinate', 'start_coordinate', 'region', 'text', 'scroll_direction', 'scroll_amount', 'duration', 'repeat'];
+const LOOKS = new Set(['screenshot', 'zoom']);
+function imageContent(img) {
+    return { type: 'image', source: { type: 'base64', media_type: img.format === 'jpeg' ? 'image/jpeg' : 'image/png', data: img.data } };
+}
+function actionOf(block) {
+    const input = (block.input !== null && typeof block.input === 'object' ? block.input : {});
+    const a = { action: block.name };
+    for (const k of PARAMS)
+        if (input[k] !== undefined && input[k] !== null)
+            a[k] = input[k];
+    return a;
+}
+/**
+ * Answers Claude's computer toolset (`tools: [COMPUTER_TOOLSET]`, Claude Opus 5.5 / Sonnet 5.5 and later on the
+ * Claude API) from this workspace's desktop:
+ *
+ *   const computer = computerToolset(workspace);
+ *   const msg = await anthropic.messages.create({ model, max_tokens, tools: [computer.definition], messages });
+ *   messages.push({ role: 'assistant', content: msg.content });
+ *   messages.push({ role: 'user', content: await computer.run(msg.content) });
+ *
+ * `run` takes the response content, runs every `toolset_name: "computer"` call of the turn as one batch (in order;
+ * the first failure stops it and the rest are answered "Not executed"), and returns one `tool_result` per call:
+ * images for screenshot and zoom, the position for cursor_position, `OK` otherwise. When the batch does not end with a
+ * look, a screenshot is attached to the last result, so the model sees the outcome without another round trip. A
+ * batch the desktop refuses as invalid (a coordinate outside the screen) is answered with the reason on every call.
+ */
+export function computerToolset(workspace) {
+    return {
+        definition: COMPUTER_TOOLSET,
+        async run(content, opts = {}) {
+            const calls = content.filter((b) => b !== null && typeof b === 'object' && b.type === 'tool_use' && b.toolset_name === 'computer');
+            if (calls.length === 0)
+                return [];
+            const lastLooks = LOOKS.has(calls[calls.length - 1].name);
+            let r;
+            try {
+                r = await workspace.computer.act(calls.map(actionOf), {
+                    screenshot: !lastLooks,
+                    ...(opts.format !== undefined ? { format: opts.format } : {}),
+                    ...(opts.quality !== undefined ? { quality: opts.quality } : {}),
+                    ...(opts.settleMs !== undefined ? { settleMs: opts.settleMs } : {}),
+                    ...(opts.signal !== undefined ? { signal: opts.signal } : {}),
+                });
+            }
+            catch (err) {
+                if (err instanceof ShardfluxApiError && err.code === 'validation_failed') {
+                    const at = typeof err.details?.index === 'number' ? err.details.index : -1;
+                    return calls.map((c, i) => ({
+                        type: 'tool_result',
+                        tool_use_id: c.id,
+                        toolset_name: 'computer',
+                        is_error: true,
+                        content: [{ type: 'text', text: i === at || at < 0 ? err.message : 'Not executed: another computer action in this turn was invalid.' }],
+                    }));
+                }
+                throw err;
+            }
+            const out = calls.map((c, i) => {
+                const res = r.results[i];
+                if (!res || res.skipped) {
+                    return { type: 'tool_result', tool_use_id: c.id, toolset_name: 'computer', is_error: true, content: [{ type: 'text', text: 'Not executed: an earlier computer action in this turn failed.' }] };
+                }
+                if (!res.ok) {
+                    return { type: 'tool_result', tool_use_id: c.id, toolset_name: 'computer', is_error: true, content: [{ type: 'text', text: res.error?.message ?? 'The action failed.' }] };
+                }
+                const body = res.image ? [imageContent(res.image)] : [{ type: 'text', text: res.output ?? 'OK' }];
+                return { type: 'tool_result', tool_use_id: c.id, toolset_name: 'computer', content: body };
+            });
+            if (r.screenshot)
+                out[out.length - 1].content.push(imageContent(r.screenshot));
+            return out;
+        },
+    };
+}