@allternit/computer-driver 0.0.0-stage → 0.2.0

package/dist/gemini.d.ts

@@ -0,0 +1,56 @@
+import { type AllternitComputers, type Approval, type ToolsetCall, type ToolsetResult } from "./client.ts";
+export interface GeminiFunctionCall {
+    name: string;
+    args?: Record<string, unknown>;
+}
+export interface GeminiAdapterOptions {
+    client: AllternitComputers;
+    computerId: string;
+    /** URL used by the `search` function. Default https://www.google.com. */
+    searchUrl?: string;
+}
+type Step = Pick<ToolsetCall, "member" | "input">;
+/** "Control+Shift+T" → "ctrl+shift+t". */
+export declare function geminiKeys(keys: string): string;
+/** Contract calls for one Gemini computer_use function call, in order. */
+export declare function geminiToCalls(fc: GeminiFunctionCall, searchUrl?: string): Step[];
+export interface GeminiStepResult {
+    /** A `functionResponse` part for the next turn. */
+    functionResponse: {
+        name: string;
+        response: Record<string, unknown>;
+    };
+    /** The screenshot after the action, as an `inlineData` part. */
+    inlineData: {
+        mimeType: string;
+        data: string;
+    };
+    error?: ToolsetResult;
+}
+/** Run one Gemini function call and return the parts Gemini expects back. */
+export declare function runGeminiCall(o: GeminiAdapterOptions, fc: GeminiFunctionCall): Promise<GeminiStepResult>;
+/**
+ * The `computer_v2` function declaration for Gemini models: the ten structured
+ * members next to the `computer_use` declaration. Route matching
+ * `functionCall` parts through runGeminiV2Call.
+ */
+export declare function geminiComputerV2Declaration(): {
+    name: string;
+    description: string;
+    parameters: Record<string, unknown>;
+};
+export interface GeminiV2AdapterOptions extends GeminiAdapterOptions {
+    /** Called when the server holds the call (409 approval_required). Without it the ApprovalRequiredError propagates, like the pixel adapter. */
+    onApproval?: (approval: Approval) => boolean | Promise<boolean>;
+}
+export interface GeminiV2StepResult {
+    /** A `functionResponse` part for the next turn. */
+    functionResponse: {
+        name: string;
+        response: Record<string, unknown>;
+    };
+    error?: ToolsetResult;
+}
+/** Run one `computer_v2` function call (`{action, ...fields}`) and return the function response part. */
+export declare function runGeminiV2Call(o: GeminiV2AdapterOptions, name: string, input: unknown): Promise<GeminiV2StepResult>;
+export {};

package/dist/gemini.js

@@ -0,0 +1,107 @@
+// Gemini computer_use adapter: runs one predefined function call on an Allternit
+// hosted computer. Gemini coordinates are 0–999 normalized, so every call is sent
+// with coordinate_space "normalized_1000" and the server scales to the screen.
+import { resultImage, resultText } from "./client.js";
+import { computerV2Tool, runComputerV2Member } from "./v2.js";
+const pt = (x, y) => [Math.round(Number(x)), Math.round(Number(y))];
+const key = (text) => ({ member: "key", input: { text } });
+const typed = (text) => ({ member: "type", input: { text } });
+const goTo = (url) => [key("ctrl+l"), typed(url), key("Return")];
+/** "Control+Shift+T" → "ctrl+shift+t". */
+export function geminiKeys(keys) {
+    const map = { control: "ctrl", meta: "super", command: "super", enter: "Return", escape: "Escape", backspace: "BackSpace", tab: "Tab", delete: "Delete" };
+    return keys.split("+").map((k) => k.trim()).map((k) => map[k.toLowerCase()] ?? (k.length === 1 ? k.toLowerCase() : k)).join("+");
+}
+/** Contract calls for one Gemini computer_use function call, in order. */
+export function geminiToCalls(fc, searchUrl = "https://www.google.com") {
+    const a = fc.args ?? {};
+    const dir = String(a.direction ?? "down");
+    switch (fc.name) {
+        case "open_web_browser":
+            return [];
+        case "wait_5_seconds":
+            return [{ member: "wait", input: { duration: 5 } }];
+        case "go_back":
+            return [key("alt+Left")];
+        case "go_forward":
+            return [key("alt+Right")];
+        case "search":
+            return goTo(searchUrl);
+        case "navigate":
+            return goTo(String(a.url ?? ""));
+        case "click_at":
+            return [{ member: "left_click", input: { coordinate: pt(a.x, a.y) } }];
+        case "hover_at":
+            return [{ member: "mouse_move", input: { coordinate: pt(a.x, a.y) } }];
+        case "type_text_at": {
+            const steps = [{ member: "left_click", input: { coordinate: pt(a.x, a.y) } }];
+            if (a.clear_before_typing !== false)
+                steps.push(key("ctrl+a"), key("BackSpace"));
+            steps.push(typed(String(a.text ?? "")));
+            if (a.press_enter !== false)
+                steps.push(key("Return"));
+            return steps;
+        }
+        case "key_combination":
+            return [key(geminiKeys(String(a.keys ?? "")))];
+        case "scroll_document":
+            return [{ member: "scroll", input: { coordinate: [500, 500], scroll_direction: dir, scroll_amount: 5 } }];
+        case "scroll_at": {
+            const amount = Math.max(1, Math.round(Number(a.magnitude ?? 800) / 160));
+            return [{ member: "scroll", input: { coordinate: pt(a.x, a.y), scroll_direction: dir, scroll_amount: amount } }];
+        }
+        case "drag_and_drop":
+            return [{ member: "left_click_drag", input: { start_coordinate: pt(a.x, a.y), coordinate: pt(a.destination_x, a.destination_y) } }];
+        default:
+            throw new Error(`Unsupported Gemini computer_use function: ${fc.name}`);
+    }
+}
+/** Run one Gemini function call and return the parts Gemini expects back. */
+export async function runGeminiCall(o, fc) {
+    const base = { toolset: "computer", coordinate_space: "normalized_1000" };
+    let error;
+    for (const step of geminiToCalls(fc, o.searchUrl)) {
+        const r = await o.client.toolset(o.computerId, { ...base, ...step });
+        if (r.is_error) {
+            error = r;
+            break;
+        }
+    }
+    const s = await o.client.toolset(o.computerId, { ...base, member: "screenshot", input: {} });
+    const img = resultImage(s);
+    if (!img)
+        throw new Error("Allternit computer returned no screenshot.");
+    const response = { url: "" };
+    if (error)
+        response.error = error.content.flatMap((b) => (b.type === "text" ? [b.text] : [])).join("\n");
+    return {
+        functionResponse: { name: fc.name, response },
+        inlineData: { mimeType: img.media_type, data: img.data },
+        ...(error ? { error } : {}),
+    };
+}
+// ------------------------------------------------------------------ computer_v2
+/**
+ * The `computer_v2` function declaration for Gemini models: the ten structured
+ * members next to the `computer_use` declaration. Route matching
+ * `functionCall` parts through runGeminiV2Call.
+ */
+export function geminiComputerV2Declaration() {
+    const t = computerV2Tool();
+    return { name: t.name, description: t.description, parameters: t.schema };
+}
+/** Run one `computer_v2` function call (`{action, ...fields}`) and return the function response part. */
+export async function runGeminiV2Call(o, name, input) {
+    const { action, ...fields } = (input ?? {});
+    if (!action)
+        throw new Error("The computer_v2 call needs an action.");
+    const base = { client: o.client, computerId: o.computerId };
+    const res = o.onApproval
+        ? await runComputerV2Member({ ...base, onApproval: o.onApproval }, action, fields)
+        : await o.client.toolset(o.computerId, { toolset: "computer", member: action, input: fields });
+    const text = resultText(res);
+    const response = { result: text };
+    if (res.is_error)
+        response.error = text || "The action failed.";
+    return { functionResponse: { name, response }, ...(res.is_error ? { error: res } : {}) };
+}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./client.ts";
+export * from "./v2.ts";
+export * from "./openai.ts";
+export * from "./gemini.ts";

package/dist/index.js

@@ -0,0 +1,7 @@
+// @allternit/computer-driver: drive Allternit hosted computers from any model loop.
+// The Anthropic toolsets live on the "@allternit/computer-driver/anthropic" subpath
+// so this entry never loads the optional @anthropic-ai/sdk peer.
+export * from "./client.js";
+export * from "./v2.js";
+export * from "./openai.js";
+export * from "./gemini.js";

package/dist/openai.d.ts

@@ -0,0 +1,98 @@
+import { type AllternitComputers, type Approval, type ToolsetCall, type ToolsetResult } from "./client.ts";
+type Pt = {
+    x: number;
+    y: number;
+};
+export type OpenAIComputerAction = {
+    type: "click";
+    x: number;
+    y: number;
+    button?: "left" | "right" | "wheel" | "back" | "forward";
+} | {
+    type: "double_click";
+    x: number;
+    y: number;
+} | {
+    type: "drag";
+    path: Pt[];
+} | {
+    type: "keypress";
+    keys: string[];
+} | {
+    type: "move";
+    x: number;
+    y: number;
+} | {
+    type: "scroll";
+    x: number;
+    y: number;
+    scroll_x: number;
+    scroll_y: number;
+} | {
+    type: "type";
+    text: string;
+} | {
+    type: "wait";
+    ms?: number;
+} | {
+    type: "screenshot";
+};
+export interface OpenAIAdapterOptions {
+    client: AllternitComputers;
+    computerId: string;
+    /** display_width / display_height you declared on the computer_use_preview tool. */
+    display?: {
+        width: number;
+        height: number;
+    };
+    /** Pixels of scroll per wheel click when converting scroll_x / scroll_y. Default 100. */
+    pixelsPerScrollClick?: number;
+}
+/** ["CTRL","C"] → "ctrl+c" (the contract's xdotool-style key text). */
+export declare function openaiKeys(keys: string[]): string;
+/** Contract calls for one OpenAI action, in order. */
+export declare function openaiToCalls(a: OpenAIComputerAction, perClick?: number): Array<Pick<ToolsetCall, "member" | "input">>;
+export interface OpenAIStepResult {
+    /** Ready for the Responses API `input`: `{type:"computer_call_output", call_id, output}`. */
+    output: {
+        type: "computer_call_output";
+        call_id: string;
+        output: {
+            type: "computer_screenshot";
+            image_url: string;
+        };
+    };
+    /** The last failed action result, if any action failed. */
+    error?: ToolsetResult;
+}
+/** Run one OpenAI `computer_call` (its `call_id` and `action`) and return the output item. */
+export declare function runOpenAIAction(o: OpenAIAdapterOptions, callId: string, action: OpenAIComputerAction): Promise<OpenAIStepResult>;
+/**
+ * The `computer_v2` function tool for the Responses API: the ten structured
+ * members next to the `computer_use_preview` tool. Register it in `tools`;
+ * route matching `function_call` items through runOpenAIV2Call.
+ */
+export declare function openaiComputerV2Tool(): {
+    type: "function";
+    name: string;
+    description: string;
+    parameters: Record<string, unknown>;
+    strict: false;
+};
+export interface OpenAIV2AdapterOptions extends OpenAIAdapterOptions {
+    /** Called when the server holds the call (409 approval_required). Without it the ApprovalRequiredError propagates, like the pixel adapter. */
+    onApproval?: (approval: Approval) => boolean | Promise<boolean>;
+}
+export interface OpenAIV2StepResult {
+    /** Ready for the Responses API `input`: `{type:"function_call_output", call_id, output}`. */
+    output: {
+        type: "function_call_output";
+        call_id: string;
+        output: string;
+    };
+    /** The failed action result, if the action failed. */
+    error?: ToolsetResult;
+}
+/** Run one `computer_v2` function call (`{action, ...fields}`) and return the `function_call_output` item. */
+export declare function runOpenAIV2Call(o: OpenAIV2AdapterOptions, callId: string, input: unknown): Promise<OpenAIV2StepResult>;
+export {};

package/dist/openai.js

@@ -0,0 +1,113 @@
+// OpenAI computer-use adapter: runs one `computer_call` action on an Allternit
+// hosted computer and returns a `computer_call_output` with the next screenshot.
+import { resultImage } from "./client.js";
+import { computerV2Tool, runComputerV2Member } from "./v2.js";
+const KEYS = {
+    CTRL: "ctrl", CONTROL: "ctrl", ALT: "alt", OPTION: "alt", SHIFT: "shift", CMD: "super", META: "super", SUPER: "super",
+    WIN: "super", ENTER: "Return", RETURN: "Return", ESC: "Escape", ESCAPE: "Escape", TAB: "Tab", SPACE: "space",
+    BACKSPACE: "BackSpace", DELETE: "Delete", UP: "Up", DOWN: "Down", LEFT: "Left", RIGHT: "Right",
+    ARROWUP: "Up", ARROWDOWN: "Down", ARROWLEFT: "Left", ARROWRIGHT: "Right", HOME: "Home", END: "End",
+    PAGEUP: "Page_Up", PAGEDOWN: "Page_Down",
+};
+/** ["CTRL","C"] → "ctrl+c" (the contract's xdotool-style key text). */
+export function openaiKeys(keys) {
+    return keys.map((k) => KEYS[k.toUpperCase()] ?? (k.length === 1 ? k.toLowerCase() : k)).join("+");
+}
+/** Contract calls for one OpenAI action, in order. */
+export function openaiToCalls(a, perClick = 100) {
+    const c = (x, y) => [Math.round(x), Math.round(y)];
+    switch (a.type) {
+        case "click": {
+            const b = a.button ?? "left";
+            if (b === "back")
+                return [{ member: "key", input: { text: "alt+Left" } }];
+            if (b === "forward")
+                return [{ member: "key", input: { text: "alt+Right" } }];
+            const member = b === "right" ? "right_click" : b === "wheel" ? "middle_click" : "left_click";
+            return [{ member, input: { coordinate: c(a.x, a.y) } }];
+        }
+        case "double_click":
+            return [{ member: "double_click", input: { coordinate: c(a.x, a.y) } }];
+        case "drag": {
+            const p = a.path;
+            if (p.length < 2)
+                return [];
+            if (p.length === 2)
+                return [{ member: "left_click_drag", input: { start_coordinate: c(p[0].x, p[0].y), coordinate: c(p[1].x, p[1].y) } }];
+            return [
+                { member: "mouse_move", input: { coordinate: c(p[0].x, p[0].y) } },
+                { member: "left_mouse_down", input: {} },
+                ...p.slice(1).map((q) => ({ member: "mouse_move", input: { coordinate: c(q.x, q.y) } })),
+                { member: "left_mouse_up", input: {} },
+            ];
+        }
+        case "keypress":
+            return [{ member: "key", input: { text: openaiKeys(a.keys) } }];
+        case "move":
+            return [{ member: "mouse_move", input: { coordinate: c(a.x, a.y) } }];
+        case "scroll": {
+            const out = [];
+            const amt = (v) => Math.max(1, Math.round(Math.abs(v) / perClick));
+            if (a.scroll_y)
+                out.push({ member: "scroll", input: { coordinate: c(a.x, a.y), scroll_direction: a.scroll_y > 0 ? "down" : "up", scroll_amount: amt(a.scroll_y) } });
+            if (a.scroll_x)
+                out.push({ member: "scroll", input: { coordinate: c(a.x, a.y), scroll_direction: a.scroll_x > 0 ? "right" : "left", scroll_amount: amt(a.scroll_x) } });
+            return out;
+        }
+        case "type":
+            return [{ member: "type", input: { text: a.text } }];
+        case "wait":
+            return [{ member: "wait", input: { duration: Math.max(1, Math.round((a.ms ?? 2000) / 1000)) } }];
+        case "screenshot":
+            return [];
+    }
+}
+/** Run one OpenAI `computer_call` (its `call_id` and `action`) and return the output item. */
+export async function runOpenAIAction(o, callId, action) {
+    const frame = o.display ? { model_frame: o.display, coordinate_space: "pixels" } : {};
+    let error;
+    for (const step of openaiToCalls(action, o.pixelsPerScrollClick)) {
+        const r = await o.client.toolset(o.computerId, { toolset: "computer", ...step, ...frame });
+        if (r.is_error) {
+            error = r;
+            break;
+        }
+    }
+    const s = await o.client.toolset(o.computerId, { toolset: "computer", member: "screenshot", input: {}, ...frame });
+    const img = resultImage(s);
+    if (!img)
+        throw new Error("Allternit computer returned no screenshot.");
+    return {
+        output: {
+            type: "computer_call_output",
+            call_id: callId,
+            output: { type: "computer_screenshot", image_url: `data:${img.media_type};base64,${img.data}` },
+        },
+        ...(error ? { error } : {}),
+    };
+}
+// ------------------------------------------------------------------ computer_v2
+/**
+ * The `computer_v2` function tool for the Responses API: the ten structured
+ * members next to the `computer_use_preview` tool. Register it in `tools`;
+ * route matching `function_call` items through runOpenAIV2Call.
+ */
+export function openaiComputerV2Tool() {
+    const t = computerV2Tool();
+    return { type: "function", name: t.name, description: t.description, parameters: t.schema, strict: false };
+}
+/** Run one `computer_v2` function call (`{action, ...fields}`) and return the `function_call_output` item. */
+export async function runOpenAIV2Call(o, callId, input) {
+    const { action, ...fields } = (input ?? {});
+    if (!action)
+        throw new Error("The computer_v2 call needs an action.");
+    const base = { client: o.client, computerId: o.computerId };
+    const res = o.onApproval
+        ? await runComputerV2Member({ ...base, onApproval: o.onApproval }, action, fields)
+        : await o.client.toolset(o.computerId, { toolset: "computer", member: action, input: fields });
+    const text = res.content.flatMap((b) => (b.type === "text" ? [b.text] : [])).join("\n");
+    return {
+        output: { type: "function_call_output", call_id: callId, output: text || "Done." },
+        ...(res.is_error ? { error: res } : {}),
+    };
+}