humanish 0.104.3 → 0.106.0

package/dist/actors/computer-use/openai-wire.js

@@ -0,0 +1,365 @@
+// The OpenAI Responses computer-use wire shape, mapped to and from the provider-neutral CuaAction
+// and CuaTurn types. Everything here is pure: the action mapper, the response parser and the
+// request-body builders touch no network and no key, so unit tests drive them directly. The live
+// client that posts these bodies is openai-provider.ts. Fixtures for this module come from
+// captured live responses; the wire-capture note in openai-provider.ts says how to take them.
+// ---------------------------------------------------------------------------
+// Defensive readers. The Responses wire shape is loosely typed (unknown), so we
+// read every field defensively: a non-object is treated as empty, a non-number
+// coordinate becomes 0, and a non-string text becomes "".
+// ---------------------------------------------------------------------------
+export function asRecord(value) {
+    return typeof value === "object" && value !== null ? value : {};
+}
+function asArray(value) {
+    return Array.isArray(value) ? value : [];
+}
+function asNumber(value) {
+    return typeof value === "number" && Number.isFinite(value) ? value : 0;
+}
+function asString(value) {
+    return typeof value === "string" ? value : "";
+}
+function optionalNumber(value) {
+    return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+export function optionalString(value) {
+    return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+function asPoint(value) {
+    const point = asRecord(value);
+    return { x: asNumber(point.x), y: asNumber(point.y) };
+}
+/** The keys a pointer action holds (`keys` on click, double_click, drag, move and scroll). */
+function heldKeysOf(record) {
+    const keys = asArray(record.keys).filter((key) => typeof key === "string");
+    return keys.length === 0 ? {} : { heldKeys: keys };
+}
+/**
+ * Map an OpenAI computer action object to a provider-neutral CuaAction, or null
+ * for an unknown type. Every coordinate and field is read defensively so a
+ * malformed action never throws and a non-number coordinate becomes 0.
+ */
+export function openAiActionToCua(action) {
+    const record = asRecord(action);
+    const type = asString(record.type);
+    switch (type) {
+        case "click": {
+            const button = record.button;
+            // The API names the middle button `wheel`. Its back and forward buttons navigate the
+            // browser's history wherever the pointer is, so they run as the same keyboard shortcuts;
+            // the executors have no mouse button 8 or 9.
+            // Held keys join the shortcut, so a held key the shortcut repeats is refused, not dropped.
+            const held = heldKeysOf(record).heldKeys ?? [];
+            if (button === "back")
+                return { kind: "keypress", keys: [...held, "ALT", "LEFT"] };
+            if (button === "forward")
+                return { kind: "keypress", keys: [...held, "ALT", "RIGHT"] };
+            return {
+                kind: "click",
+                x: asNumber(record.x),
+                y: asNumber(record.y),
+                button: button === "right"
+                    ? "right"
+                    : button === "wheel" || button === "middle"
+                        ? "middle"
+                        : "left",
+                ...heldKeysOf(record),
+            };
+        }
+        case "double_click":
+            return {
+                kind: "double_click",
+                x: asNumber(record.x),
+                y: asNumber(record.y),
+                ...heldKeysOf(record),
+            };
+        case "move":
+            return { kind: "move", x: asNumber(record.x), y: asNumber(record.y), ...heldKeysOf(record) };
+        case "scroll":
+            return {
+                kind: "scroll",
+                x: asNumber(record.x),
+                y: asNumber(record.y),
+                dx: asNumber(record.scroll_x),
+                dy: asNumber(record.scroll_y),
+                ...heldKeysOf(record),
+            };
+        case "type":
+            return { kind: "type", text: asString(record.text) };
+        case "keypress":
+            return {
+                kind: "keypress",
+                keys: asArray(record.keys).filter((key) => typeof key === "string"),
+            };
+        case "drag":
+            return { kind: "drag", path: asArray(record.path).map(asPoint), ...heldKeysOf(record) };
+        case "wait":
+            return { kind: "wait" };
+        case "screenshot":
+            return { kind: "screenshot" };
+        default:
+            return null;
+    }
+}
+// Collect plain strings or { text } entries from a reasoning summary/content array.
+function collectTextEntries(value) {
+    const out = [];
+    for (const entry of asArray(value)) {
+        if (typeof entry === "string") {
+            if (entry.length > 0)
+                out.push(entry);
+            continue;
+        }
+        const text = asString(asRecord(entry).text);
+        if (text.length > 0)
+            out.push(text);
+    }
+    return out;
+}
+// Collect output_text strings from a message content array.
+function collectMessageText(value) {
+    const out = [];
+    for (const entry of asArray(value)) {
+        const record = asRecord(entry);
+        if (asString(record.type) === "output_text") {
+            const text = asString(record.text);
+            if (text.length > 0)
+                out.push(text);
+        }
+    }
+    return out;
+}
+/**
+ * Parse a Responses API response into a provider-neutral CuaTurn plus the
+ * computer_call ids (needed to build the next turn's call outputs) and the raw
+ * output items (needed for ZDR explicit-context continuation). Pure: no network,
+ * no mutation of the input. Optional CuaTurn fields are only set when present so
+ * the result satisfies exactOptionalPropertyTypes.
+ */
+export function parseOpenAiResponse(raw) {
+    const root = asRecord(raw);
+    const responseId = optionalString(root.id);
+    const output = asArray(root.output);
+    const actions = [];
+    const callIds = [];
+    const reasoningParts = [];
+    const messageParts = [];
+    const safetyChecks = [];
+    for (const rawItem of output) {
+        const item = asRecord(rawItem);
+        switch (asString(item.type)) {
+            case "reasoning":
+                reasoningParts.push(...collectTextEntries(item.summary), ...collectTextEntries(item.content));
+                break;
+            case "message":
+                messageParts.push(...collectMessageText(item.content));
+                break;
+            case "output_text": {
+                const text = asString(item.text);
+                if (text.length > 0)
+                    messageParts.push(text);
+                break;
+            }
+            case "computer_call": {
+                const callId = optionalString(item.call_id);
+                if (callId !== undefined)
+                    callIds.push(callId);
+                // The live Responses API returns the actions as an ARRAY (`item.actions`); a single
+                // computer_call can carry several. (An older/alt shape used a singular `item.action` —
+                // supported as a fallback.) Reading only `item.action` silently dropped EVERY action,
+                // which made the loop see zero actions and stop on a false `goal_satisfied`.
+                const rawActions = Array.isArray(item.actions)
+                    ? item.actions
+                    : item.action !== undefined
+                        ? [item.action]
+                        : [];
+                for (const rawAction of rawActions) {
+                    const mapped = openAiActionToCua(rawAction);
+                    if (mapped !== null)
+                        actions.push(mapped);
+                }
+                // Preserve the wire triple verbatim: the API matches acknowledgements on
+                // `id`, so collapsing to a code string (and fabricating ids on echo)
+                // would silently break the proceed path.
+                for (const rawCheck of asArray(item.pending_safety_checks)) {
+                    const check = asRecord(rawCheck);
+                    const id = asString(check.id);
+                    const code = asString(check.code);
+                    safetyChecks.push({
+                        id: id || code || "safety_check",
+                        code: code || id || "safety_check",
+                        message: asString(check.message) || code || id || "safety_check",
+                    });
+                }
+                break;
+            }
+            default:
+                break;
+        }
+    }
+    const topText = asString(root.output_text);
+    if (topText.length > 0)
+        messageParts.push(topText);
+    const reasoning = reasoningParts.filter((part) => part.length > 0).join("\n");
+    const message = messageParts.filter((part) => part.length > 0).join("\n");
+    const usageRecord = asRecord(root.usage);
+    const usageInput = optionalNumber(usageRecord.input_tokens);
+    const usageOutput = optionalNumber(usageRecord.output_tokens);
+    // Of the input tokens, how many the provider served from its prompt cache. This loop threads
+    // state with previous_response_id and re-sends a growing warm prefix every turn, so most input on
+    // a long session is a cache hit billed at a fraction of the full rate. Not reading it made every
+    // cost line materially overstate the bill (#391).
+    const usageCachedInput = optionalNumber(asRecord(usageRecord.input_tokens_details).cached_tokens);
+    // GPT-5.6+ bills cache WRITES (1.25x input) and reports them here; older models omit the field.
+    const usageCacheWriteInput = optionalNumber(asRecord(usageRecord.input_tokens_details).cache_write_tokens);
+    const usage = usageInput === undefined && usageOutput === undefined
+        ? undefined
+        : {
+            ...(usageInput === undefined ? {} : { input: usageInput }),
+            ...(usageOutput === undefined ? {} : { output: usageOutput }),
+            ...(usageCachedInput === undefined ? {} : { cachedInput: usageCachedInput }),
+            ...(usageCacheWriteInput === undefined ? {} : { cacheWriteInput: usageCacheWriteInput }),
+        };
+    // Responses can exhaust output/context tokens before producing any visible answer. Empty
+    // actions on that wire status are an interrupted generation, never a natural endpoint.
+    const interruption = root.status === "incomplete"
+        ? asRecord(root.incomplete_details).reason === "max_output_tokens"
+            ? "output_limit"
+            : "incomplete"
+        : root.status !== undefined && root.status !== "completed"
+            ? "unexpected_status"
+            : undefined;
+    const turn = {
+        actions,
+        pendingSafetyChecks: safetyChecks,
+        done: interruption === undefined && actions.length === 0,
+        ...(interruption === undefined ? {} : { interruption }),
+        ...(responseId === undefined ? {} : { responseId }),
+        ...(reasoning.length > 0 ? { reasoning } : {}),
+        ...(message.length > 0 ? { message } : {}),
+        ...(usage === undefined ? {} : { usage }),
+    };
+    return { turn, callIds, outputItems: output };
+}
+// The fields shared by the initial and continuation requests: the tool spec,
+// truncation policy, reasoning effort, and (when configured) the safety id.
+function sharedRequestFields(ctx) {
+    return {
+        model: ctx.model,
+        ...(ctx.maxOutputTokens === undefined ? {} : { max_output_tokens: ctx.maxOutputTokens }),
+        // Keep the task/persona contract present on every turn. Some computer-use
+        // continuations carry only screenshot call outputs; without repeating the
+        // instructions, a provider that does not fully retain prior state can drift
+        // into asking the operator what to do.
+        instructions: ctx.instructions,
+        // The Responses API `computer` tool takes no display/environment fields — the model infers
+        // resolution from the screenshots it is sent. (Sending display_* returns a 400
+        // "Unknown parameter tools[0].display_width", confirmed against the live API 2026-06.)
+        tools: [{ type: "computer" }],
+        truncation: "auto",
+        // `summary` asks for the provider-SANCTIONED reasoning summary items (#427) — the
+        // capture side never scrapes or reconstructs raw chain-of-thought. Parsed by
+        // parseOpenAiResponse into turn.reasoning; the loop records them as redacted
+        // `kind: "reasoning"` trace items.
+        reasoning: {
+            effort: ctx.reasoningEffort,
+            ...(ctx.reasoningSummary === undefined ? {} : { summary: ctx.reasoningSummary }),
+        },
+        ...(ctx.safetyIdentifier === undefined ? {} : { safety_identifier: ctx.safetyIdentifier }),
+    };
+}
+/**
+ * The first request's output limit. A first request can spend its whole allowance inside an
+ * unfinished computer_call; this bound ends that within seconds while leaving several times the
+ * room a completed first turn uses (a screenshot request of a few dozen tokens).
+ */
+const FIRST_TURN_OUTPUT_LIMIT = 1024;
+/**
+ * Build the first-turn request body: the instructions and the first screen, so the first decision
+ * is made with the page in view. Its output limit is FIRST_TURN_OUTPUT_LIMIT, or the declared
+ * limit when that is lower.
+ */
+export function buildInitialRequest(ctx, screenshot) {
+    const maxOutputTokens = Math.min(ctx.maxOutputTokens ?? FIRST_TURN_OUTPUT_LIMIT, FIRST_TURN_OUTPUT_LIMIT);
+    return {
+        ...sharedRequestFields({ ...ctx, maxOutputTokens }),
+        input: [
+            {
+                role: "user",
+                content: [
+                    { type: "input_text", text: ctx.instructions },
+                    ...(screenshot === undefined
+                        ? []
+                        : [
+                            {
+                                type: "input_image",
+                                image_url: `data:image/png;base64,${screenshot.toString("base64")}`,
+                            },
+                        ]),
+                ],
+            },
+        ],
+    };
+}
+/**
+ * Build one computer_call_output for a pending call id, carrying the latest
+ * screenshot as an inline data URL. Acknowledged safety checks (if any) are
+ * echoed back so the model can proceed past a check the harness approved.
+ *
+ * The screenshot param is `Buffer | undefined` because CuaObservation.screenshot is now
+ * optional (a non-vision executor omits it). This provider is a VISION model (it sets
+ * requiresFrame), so a missing frame is a hard error here — defense-in-depth: the loop's
+ * per-turn requiresFrame guard already fails closed before this is reached, but throwing keeps
+ * the mapper self-validating and isolable.
+ */
+export function buildCallOutput(callId, screenshot, acknowledged) {
+    if (screenshot === undefined) {
+        throw new Error("openai-responses-cu requires observation.screenshot (it is a vision provider; pair a state-only executor with a non-vision provider)");
+    }
+    return {
+        type: "computer_call_output",
+        call_id: callId,
+        output: {
+            type: "computer_screenshot",
+            image_url: `data:image/png;base64,${screenshot.toString("base64")}`,
+        },
+        ...(acknowledged && acknowledged.length > 0
+            ? {
+                acknowledged_safety_checks: acknowledged.map(({ id, code, message }) => ({
+                    id,
+                    code,
+                    message,
+                })),
+            }
+            : {}),
+    };
+}
+// Turn an optional context-hint string into an input item array (or empty).
+function hintItems(contextHint) {
+    return contextHint
+        ? [{ role: "user", content: [{ type: "input_text", text: contextHint }] }]
+        : [];
+}
+/**
+ * Build a continuation request body. Two modes:
+ *  - default: thread server-side state via previous_response_id and send only the
+ *    new call outputs (plus an optional hint).
+ *  - explicit-context (ZDR): no previous_response_id; the prior output items are
+ *    re-sent inline ahead of the new call outputs so the model has full context
+ *    without the server retaining any.
+ */
+export function buildContinuationRequest(args) {
+    const { ctx, previousResponseId, callOutputs, contextHint, explicitContextItems } = args;
+    if (explicitContextItems === undefined) {
+        return {
+            ...sharedRequestFields(ctx),
+            previous_response_id: previousResponseId,
+            input: [...callOutputs, ...hintItems(contextHint)],
+        };
+    }
+    return {
+        ...sharedRequestFields(ctx),
+        input: [...explicitContextItems, ...callOutputs, ...hintItems(contextHint)],
+    };
+}

package/dist/actors/computer-use/provider-error.d.ts

@@ -0,0 +1,27 @@
+import type { ActorTokenUsage, ProviderRequestReceipt } from "../contract.js";
+declare const CUA_PROVIDER_FAILURE_PHASES: readonly ["startup", "initialize", "config/read", "account/read", "thread/start", "mcpServerStatus/list", "turn/start", "response", "cleanup"];
+export type CuaProviderFailurePhase = (typeof CUA_PROVIDER_FAILURE_PHASES)[number];
+export declare const isCuaProviderFailurePhase: (value: unknown) => value is CuaProviderFailurePhase;
+declare const codes: readonly ["request_rejected", "unavailable", "busy", "refused", "invalid_response", "protocol_error", "timeout", "cancelled", "process_failed", "cleanup_unconfirmed"];
+export type CuaProviderErrorCode = (typeof codes)[number];
+/** Safe local classification. Never attach provider prose, stderr, paths or a raw cause. */
+export declare class CuaProviderError extends Error {
+    readonly code: CuaProviderErrorCode;
+    readonly receipt: ProviderRequestReceipt;
+    readonly usage?: ActorTokenUsage;
+    readonly failurePhase?: CuaProviderFailurePhase;
+    constructor(code: CuaProviderErrorCode, receipt: ProviderRequestReceipt, usage?: ActorTokenUsage, failurePhase?: CuaProviderFailurePhase);
+}
+export declare const isCuaProviderError: (value: unknown) => value is CuaProviderError;
+/**
+ * The model provider refused the request under its usage policy (OpenAI `invalid_prompt`). A
+ * refusal is the provider's decision about this prompt, so it is not a harness fault, and sending
+ * the same prompt again is not a recovery path. `providerCode` is the provider's identifier only;
+ * the provider's message can echo the prompt and is never attached.
+ */
+export declare class CuaPromptRefusedError extends Error {
+    readonly providerCode: string;
+    constructor(provider: string, providerCode: string);
+}
+export declare const isCuaPromptRefusedError: (value: unknown) => value is CuaPromptRefusedError;
+export {};

package/dist/actors/computer-use/provider-error.js

@@ -0,0 +1,86 @@
+const CUA_PROVIDER_FAILURE_PHASES = [
+    "startup",
+    "initialize",
+    "config/read",
+    "account/read",
+    "thread/start",
+    "mcpServerStatus/list",
+    "turn/start",
+    "response",
+    "cleanup",
+];
+export const isCuaProviderFailurePhase = (value) => typeof value === "string" && CUA_PROVIDER_FAILURE_PHASES.includes(value);
+const codes = Object.freeze([
+    "request_rejected",
+    "unavailable",
+    "busy",
+    "refused",
+    "invalid_response",
+    "protocol_error",
+    "timeout",
+    "cancelled",
+    "process_failed",
+    "cleanup_unconfirmed",
+]);
+const errors = new WeakSet();
+/** Safe local classification. Never attach provider prose, stderr, paths or a raw cause. */
+export class CuaProviderError extends Error {
+    code;
+    receipt;
+    usage;
+    failurePhase;
+    constructor(code, receipt, usage, failurePhase) {
+        if (typeof code !== "string" ||
+            !codes.includes(code) ||
+            !receipt ||
+            typeof receipt !== "object" ||
+            Object.keys(receipt).length !== 3 ||
+            !(typeof receipt.dispatched === "boolean" || receipt.dispatched === "unknown") ||
+            typeof receipt.usageComplete !== "boolean" ||
+            (receipt.cleanup !== "confirmed" && receipt.cleanup !== "unconfirmed") ||
+            (failurePhase !== undefined && !isCuaProviderFailurePhase(failurePhase)) ||
+            (usage !== undefined &&
+                (!usage ||
+                    typeof usage !== "object" ||
+                    Array.isArray(usage) ||
+                    Object.entries(usage).some(([key, count]) => !["input", "output", "cachedInput", "cacheWriteInput", "total"].includes(key) ||
+                        typeof count !== "number" ||
+                        !Number.isSafeInteger(count) ||
+                        count < 0)))) {
+            throw new TypeError("Invalid participant provider error declaration.");
+        }
+        super(`Participant provider: ${code}`);
+        this.name = "CuaProviderError";
+        this.code = code;
+        this.receipt = Object.freeze({
+            dispatched: receipt.dispatched,
+            usageComplete: receipt.usageComplete,
+            cleanup: receipt.cleanup,
+        });
+        if (usage !== undefined)
+            this.usage = Object.freeze({ ...usage });
+        if (failurePhase !== undefined)
+            this.failurePhase = failurePhase;
+        for (const key of ["code", "receipt", "usage", "failurePhase"])
+            Object.defineProperty(this, key, { writable: false, configurable: false });
+        errors.add(this);
+    }
+}
+export const isCuaProviderError = (value) => typeof value === "object" && value !== null && errors.has(value);
+const refusals = new WeakSet();
+/**
+ * The model provider refused the request under its usage policy (OpenAI `invalid_prompt`). A
+ * refusal is the provider's decision about this prompt, so it is not a harness fault, and sending
+ * the same prompt again is not a recovery path. `providerCode` is the provider's identifier only;
+ * the provider's message can echo the prompt and is never attached.
+ */
+export class CuaPromptRefusedError extends Error {
+    providerCode;
+    constructor(provider, providerCode) {
+        super(`${provider} refused the prompt under its usage policy (${providerCode})`);
+        this.name = "CuaPromptRefusedError";
+        this.providerCode = providerCode;
+        refusals.add(this);
+    }
+}
+export const isCuaPromptRefusedError = (value) => typeof value === "object" && value !== null && refusals.has(value);

package/dist/actors/computer-use/speech.js

@@ -0,0 +1,6 @@
+export const CUA_SPEECH_LIMITS = Object.freeze({
+    characters: 400,
+    bytes: 1600,
+    utterances: 4,
+    durationMs: 120_000,
+});