@loadmill/droid-cua 2.23.0 → 2.25.0

package/README.md

@@ -134,6 +134,32 @@ Install:
 npm install -g @loadmill/droid-cua
 ```
 
+Install the bundled Agent Skill for your coding agent:
+
+In the desktop app, choose **Help → Install Droid CUA Skill**, then select your
+coding agent from the submenu. The app installs the skill globally for your user account.
+
+Alternatively, install it from the CLI:
+
+```sh
+# Global installation (default)
+droid-cua skill install cursor
+droid-cua skill install claude-code
+droid-cua skill install codex
+
+# Or install in the current project
+droid-cua skill install codex --local
+```
+
+Use `droid-cua skill status <agent>` to inspect an installation and
+`droid-cua skill uninstall <agent>` to remove it. Installed skill
+files are never overwritten or removed after local modification unless `--force`
+is supplied. After upgrading the npm package, update an installed skill explicitly:
+
+```sh
+droid-cua skill install <agent> --force
+```
+
 Examples:
 ```sh
 # Interactive CLI
@@ -230,6 +256,10 @@ Supported CLI options include:
 - `--report`
 - `--debug`
 
+`--cua-model` accepts `loadmill-smart`, `loadmill-beacon`, `gpt-5.6-terra`,
+`gpt-5.6-luna`, and `gpt-5.4`. OpenAI models use direct OpenAI mode and require
+`OPENAI_API_KEY`.
+
 Config and precedence rules:
 - Use `--config <file>` to supply prompt-affecting settings for headless runs.
 - CLI flags override config file values.

package/build/index.js

@@ -29,8 +29,20 @@ import { createStatusLine } from "./src/cli/cloud-status-line.js";
 import { createLoadmillApiClient } from "./src/integrations/loadmill/client.js";
 import { getCloudProviderAdapter } from "./src/device/cloud/registry.js";
 import { setCloudProgressListener } from "./src/device/cloud/progress.js";
+import { runSkillCommand } from "./src/cli/skill-command.js";
 dotenv.config();
 const args = minimist(process.argv.slice(2));
+if (args._[0] === "skill") {
+    try {
+        await runSkillCommand({ args });
+        process.exit(0);
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        console.error(`\nSkill command failed: ${message}`);
+        process.exit(1);
+    }
+}
 const targetConfig = resolveHeadlessTarget(args);
 let avdName = targetConfig.avdName;
 let platform = targetConfig.platform || null; // 'ios' or 'android'

package/build/src/cli/headless-runner.js

@@ -86,6 +86,7 @@ function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, executionP
         toolCapabilities: {
             assert: true,
             navigate: isBrowserTarget,
+            closeKeyboard: !isBrowserTarget,
             loadmill: true,
         },
         secretsByKey: secrets?.secretsByKey || {},
@@ -174,6 +175,7 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
             aimAssistMode: headlessConfig.aimAssistMode,
             testDataContext,
             availableSecretKeys: secrets?.availableSecretKeys || [],
+            cuaModel: runtimeContext.config.cuaModel,
         });
         session.setSystemPrompt(executionPrompt);
         const screenshotRecorder = headlessDebug.createExecutionScreenshotRecorder({

package/build/src/cli/skill-command.js

@@ -0,0 +1,240 @@
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { cp, lstat, mkdir, readdir, readFile, rm } from "node:fs/promises";
+const SKILL_NAME = "droid-cua";
+const AGENTS = Object.freeze({
+    cursor: { displayName: "Cursor", relativePath: path.join(".cursor", "skills", SKILL_NAME) },
+    "claude-code": { displayName: "Claude Code", relativePath: path.join(".claude", "skills", SKILL_NAME) },
+    codex: { displayName: "Codex", relativePath: path.join(".agents", "skills", SKILL_NAME) },
+});
+const SUBCOMMANDS = new Set(["install", "status", "uninstall"]);
+const ALLOWED_OPTIONS = new Set(["_", "local", "force", "h", "help"]);
+const COMMAND_DESCRIPTIONS = Object.freeze({
+    install: "Install Droid CUA skill for a coding agent",
+    status: "Show Droid CUA skill status for a coding agent",
+    uninstall: "Uninstall Droid CUA skill for a coding agent",
+});
+function skillCommandHelp(subcommand) {
+    const localDescriptions = {
+        install: "Install in current project instead of global",
+        status: "Check current project instead of global",
+        uninstall: "Uninstall from current project instead of global",
+    };
+    const options = [
+        `  --local      ${localDescriptions[subcommand]}`,
+    ];
+    if (subcommand !== "status") {
+        options.push("  --force      Overwrite or remove modified skill files");
+    }
+    options.push("  -h, --help  display help for command");
+    return [
+        `Usage: droid-cua skill ${subcommand} [options] [agent]`,
+        "",
+        COMMAND_DESCRIPTIONS[subcommand],
+        "",
+        "Agents:",
+        "  cursor",
+        "  claude-code",
+        "  codex",
+        "",
+        "Example:",
+        `  droid-cua skill ${subcommand} claude-code`,
+        "",
+        "Options:",
+        ...options,
+    ].join("\n");
+}
+function skillRootHelp() {
+    return [
+        "Usage: droid-cua skill <command> [options] [agent]",
+        "",
+        "Manage the bundled Droid CUA skill",
+        "",
+        "Commands:",
+        "  install    Install the skill",
+        "  status     Show installation status",
+        "  uninstall  Uninstall the skill",
+    ].join("\n");
+}
+async function pathType(targetPath) {
+    try {
+        const stats = await lstat(targetPath);
+        if (stats.isDirectory())
+            return "directory";
+        if (stats.isFile())
+            return "file";
+        if (stats.isSymbolicLink())
+            return "symlink";
+        return "other";
+    }
+    catch (error) {
+        if (error?.code === "ENOENT")
+            return "missing";
+        throw error;
+    }
+}
+async function snapshotDirectory(rootPath, relativePath = "") {
+    const absolutePath = relativePath ? path.join(rootPath, relativePath) : rootPath;
+    const entries = await readdir(absolutePath, { withFileTypes: true });
+    const snapshot = [];
+    for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
+        const childRelativePath = relativePath ? path.join(relativePath, entry.name) : entry.name;
+        if (entry.isDirectory()) {
+            snapshot.push({ path: `${childRelativePath}/`, type: "directory" });
+            snapshot.push(...await snapshotDirectory(rootPath, childRelativePath));
+        }
+        else if (entry.isFile()) {
+            const contents = await readFile(path.join(rootPath, childRelativePath));
+            snapshot.push({ path: childRelativePath, type: "file", contents: contents.toString("base64") });
+        }
+        else {
+            snapshot.push({ path: childRelativePath, type: entry.isSymbolicLink() ? "symlink" : "other" });
+        }
+    }
+    return snapshot;
+}
+async function directoryMatches(sourcePath, destinationPath) {
+    if (await pathType(destinationPath) !== "directory")
+        return false;
+    const [sourceSnapshot, destinationSnapshot] = await Promise.all([
+        snapshotDirectory(sourcePath),
+        snapshotDirectory(destinationPath),
+    ]);
+    return JSON.stringify(sourceSnapshot) === JSON.stringify(destinationSnapshot);
+}
+async function getInstallationState(sourcePath, destinationPath) {
+    if (await pathType(destinationPath) === "missing")
+        return "not-installed";
+    return await directoryMatches(sourcePath, destinationPath) ? "installed" : "modified";
+}
+function normalizeStringOption(value, label) {
+    if (typeof value !== "string" || value.trim().length === 0) {
+        throw new Error(`${label} requires a value.`);
+    }
+    return value.trim().toLowerCase();
+}
+function parseSkillCommandArgs(args) {
+    const unknownOption = Object.keys(args).find((key) => !ALLOWED_OPTIONS.has(key));
+    if (unknownOption)
+        throw new Error(`Unknown skill option --${unknownOption}.`);
+    if (!Array.isArray(args._) || args._[0] !== "skill") {
+        throw new Error("Usage: droid-cua skill <command> [options] [agent]");
+    }
+    if (args._.length === 1) {
+        return { help: skillRootHelp() };
+    }
+    const subcommand = normalizeStringOption(args._[1], "Skill subcommand");
+    if (!SUBCOMMANDS.has(subcommand)) {
+        throw new Error(`Unknown skill subcommand "${subcommand}". Expected install, status, or uninstall.`);
+    }
+    for (const option of ["local", "force", "h", "help"]) {
+        if (args[option] != null && typeof args[option] !== "boolean") {
+            throw new Error(`--${option === "h" ? "help" : option} does not accept a value.`);
+        }
+    }
+    if (args.help === true || args.h === true || args._.length === 2) {
+        return { help: skillCommandHelp(subcommand) };
+    }
+    if (args._.length !== 3)
+        throw new Error(skillCommandHelp(subcommand));
+    if (subcommand === "status" && args.force != null) {
+        throw new Error("--force is only supported by install and uninstall.");
+    }
+    const agent = normalizeStringOption(args._[2], "Agent");
+    if (!Object.hasOwn(AGENTS, agent)) {
+        throw new Error(`Unknown agent "${agent}". Expected cursor, claude-code, or codex.`);
+    }
+    return {
+        subcommand,
+        agent,
+        local: args.local === true,
+        force: args.force === true,
+    };
+}
+export function resolveSkillDestination({ agent, local = false, cwd = process.cwd(), home = os.homedir() }) {
+    const basePath = local ? cwd : home;
+    return {
+        agent,
+        ...AGENTS[agent],
+        path: path.join(basePath, AGENTS[agent].relativePath),
+    };
+}
+export async function resolveBundledSkillPath({ startDir = path.dirname(fileURLToPath(import.meta.url)) } = {}) {
+    let currentPath = path.resolve(startDir);
+    while (true) {
+        const candidate = path.join(currentPath, "skill", SKILL_NAME);
+        if (await pathType(path.join(candidate, "SKILL.md")) === "file")
+            return candidate;
+        const parentPath = path.dirname(currentPath);
+        if (parentPath === currentPath)
+            break;
+        currentPath = parentPath;
+    }
+    throw new Error("Bundled Droid CUA skill was not found. Reinstall @loadmill/droid-cua and try again.");
+}
+function stateLabel(state) {
+    if (state === "not-installed")
+        return "not installed";
+    return state;
+}
+function shouldUseColor() {
+    return Boolean(process.stdout.isTTY) && !Object.hasOwn(process.env, "NO_COLOR");
+}
+function formatInstallSuccess(destination, { alreadyInstalled, color }) {
+    const checkmark = color ? "\u001b[32m✓\u001b[39m" : "✓";
+    const skillFilePath = path.join(destination.path, "SKILL.md");
+    const displayedPath = color ? `\u001b[2m${skillFilePath}\u001b[22m` : skillFilePath;
+    const message = alreadyInstalled
+        ? `${destination.agent} skill already installed`
+        : `Installed ${destination.agent} skill`;
+    return `${checkmark} ${message}\n${displayedPath}`;
+}
+export async function runSkillCommand({ args, cwd = process.cwd(), home = os.homedir(), sourcePath = null, output = console, color = shouldUseColor(), } = {}) {
+    const command = parseSkillCommandArgs(args ?? {});
+    if (command.help) {
+        output.log(command.help);
+        return [];
+    }
+    const bundledSkillPath = sourcePath ?? await resolveBundledSkillPath();
+    if (await pathType(path.join(bundledSkillPath, "SKILL.md")) !== "file") {
+        throw new Error(`Invalid bundled skill at ${bundledSkillPath}: SKILL.md is missing.`);
+    }
+    const destination = resolveSkillDestination({ agent: command.agent, local: command.local, cwd, home });
+    const states = [{
+            ...destination,
+            state: await getInstallationState(bundledSkillPath, destination.path),
+        }];
+    if (command.subcommand !== "status" && !command.force) {
+        const conflicts = states.filter(({ state }) => state === "modified");
+        if (conflicts.length > 0) {
+            const paths = conflicts.map(({ path: destinationPath }) => destinationPath).join(", ");
+            throw new Error(`Refusing to ${command.subcommand} modified skill files at ${paths}. Re-run with --force to replace them.`);
+        }
+    }
+    for (const destination of states) {
+        if (command.subcommand === "status") {
+            output.log(`${destination.displayName}: ${stateLabel(destination.state)} at ${destination.path}`);
+        }
+        else if (command.subcommand === "install") {
+            if (destination.state === "installed") {
+                output.log(formatInstallSuccess(destination, { alreadyInstalled: true, color }));
+            }
+            else {
+                if (destination.state !== "not-installed")
+                    await rm(destination.path, { recursive: true, force: true });
+                await mkdir(path.dirname(destination.path), { recursive: true });
+                await cp(bundledSkillPath, destination.path, { recursive: true });
+                output.log(formatInstallSuccess(destination, { alreadyInstalled: false, color }));
+            }
+        }
+        else if (destination.state === "not-installed") {
+            output.log(`${destination.displayName}: not installed at ${destination.path}`);
+        }
+        else {
+            await rm(destination.path, { recursive: true, force: true });
+            output.log(`${destination.displayName}: uninstalled from ${destination.path}`);
+        }
+    }
+    return states.map(({ agent, path: destinationPath, state }) => ({ agent, path: destinationPath, state }));
+}

package/build/src/commands/help.js

@@ -1,6 +1,7 @@
 /**
  * Help command handler
  */
+import { CUA_MODELS } from '../core/cua-models.js';
 /**
  * Handle /help command
  * @param {string} args - Command arguments
@@ -32,7 +33,7 @@ export async function handleHelp(args, session, context) {
     addOutput({ type: 'info', text: '  --session-mode <mode>  Browser session mode for web runs: persistent or fresh' });
     addOutput({ type: 'info', text: '  --config <file>        Headless execution JSON config for prompt parity' });
     addOutput({ type: 'info', text: '  --llm-provider <mode>  Headless AI provider: loadmill or openai' });
-    addOutput({ type: 'info', text: '  --cua-model <model>    Headless CUA model override: gpt-5.4, loadmill-smart, loadmill-beacon, or gpt-5.4-mini' });
+    addOutput({ type: 'info', text: `  --cua-model <model>    Headless CUA model override: ${CUA_MODELS.join(', ')}` });
     addOutput({ type: 'info', text: '  --aim-assist <mode>    Headless Aim Assist mode: off, smart, or always' });
     addOutput({ type: 'info', text: '  --context <file>       Optional app context file injected into execution runs' });
     addOutput({ type: 'info', text: '  --no-context           Disable app context for headless execution' });

package/build/src/commands/run.js

@@ -67,8 +67,7 @@ export async function handleRun(args, session, context) {
     if (context.setAgentWorking) {
         context.setAgentWorking(true, 'Executing test...');
     }
-    // Reset session state to prevent context leak from design mode
-    // Each test instruction should execute in isolation
+    // Reset session state to prevent context leak from design mode.
     session.updateResponseId(undefined);
     session.clearMessages();
     const appContext = await resolveExecutionAppContext({ contextPath: context.appContextPath });
@@ -89,7 +88,10 @@ export async function handleRun(args, session, context) {
     }
     // Set execution mode system prompt (replaces any design mode prompt)
     const availableSecretKeys = context.engine.getAvailableSecretKeys();
-    const executionPrompt = buildExecutionModePrompt(session.deviceInfo, {}, appContextText, { availableSecretKeys });
+    const executionPrompt = buildExecutionModePrompt(session.deviceInfo, {}, appContextText, {
+        availableSecretKeys,
+        cuaModel: context.runtimeConfig?.cuaModel,
+    });
     session.setSystemPrompt(executionPrompt);
     // Create execution mode
     const executionMode = new ExecutionMode(session, context.engine, instructions);

package/build/src/core/cua-models.js

@@ -1,9 +1,10 @@
 export const DEFAULT_OPENAI_CUA_MODEL = "gpt-5.4";
 export const DEFAULT_LOADMILL_CUA_MODEL = "loadmill-smart";
 export const DEFAULT_CUA_MODEL = DEFAULT_LOADMILL_CUA_MODEL;
-export const OPENAI_CUA_MODELS = ["gpt-5.4", "gpt-5.4-mini"];
-export const LOADMILL_CUA_MODELS = [DEFAULT_LOADMILL_CUA_MODEL, "loadmill-beacon"];
-export const CUA_MODELS = ["gpt-5.4", "loadmill-smart", "loadmill-beacon", "gpt-5.4-mini"];
+export const CUA_CONVERSATION_POLICIES = Object.freeze({
+    CHAINED: "chained",
+    ISOLATED: "isolated",
+});
 export const CUA_MODEL_OPTIONS = Object.freeze([
     Object.freeze({
         id: "loadmill-smart",
@@ -20,20 +21,34 @@ export const CUA_MODEL_OPTIONS = Object.freeze([
         label: "Loadmill - Beacon",
     }),
     Object.freeze({
-        id: "gpt-5.4",
+        id: "gpt-5.6-terra",
         providerMode: "openai",
         providerLabel: "OpenAI",
-        name: "GPT-5.4",
-        label: "OpenAI GPT-5.4",
+        name: "GPT-5.6 Terra",
+        label: "OpenAI GPT-5.6 Terra",
     }),
     Object.freeze({
-        id: "gpt-5.4-mini",
+        id: "gpt-5.6-luna",
         providerMode: "openai",
         providerLabel: "OpenAI",
-        name: "GPT-5.4 mini",
-        label: "OpenAI GPT-5.4 mini (faster)",
+        name: "GPT-5.6 Luna",
+        label: "OpenAI GPT-5.6 Luna",
+    }),
+    Object.freeze({
+        id: "gpt-5.4",
+        providerMode: "openai",
+        providerLabel: "OpenAI",
+        name: "GPT-5.4",
+        label: "OpenAI GPT-5.4",
     }),
 ]);
+export const CUA_MODELS = Object.freeze(CUA_MODEL_OPTIONS.map((option) => option.id));
+export const OPENAI_CUA_MODELS = Object.freeze(CUA_MODEL_OPTIONS
+    .filter((option) => option.providerMode === "openai")
+    .map((option) => option.id));
+export const LOADMILL_CUA_MODELS = Object.freeze(CUA_MODEL_OPTIONS
+    .filter((option) => option.providerMode === "loadmill")
+    .map((option) => option.id));
 const VALID_CUA_MODELS = new Set(CUA_MODELS);
 const OPENAI_CUA_MODEL_SET = new Set(OPENAI_CUA_MODELS);
 const LOADMILL_CUA_MODEL_SET = new Set(LOADMILL_CUA_MODELS);
@@ -59,3 +74,8 @@ export function resolveDefaultCuaModelForProvider(providerMode) {
         ? DEFAULT_LOADMILL_CUA_MODEL
         : DEFAULT_OPENAI_CUA_MODEL;
 }
+export function resolveConversationPolicy(cuaModel) {
+    return cuaModel === "loadmill-beacon" || cuaModel === "beacon"
+        ? CUA_CONVERSATION_POLICIES.ISOLATED
+        : CUA_CONVERSATION_POLICIES.CHAINED;
+}

package/build/src/core/droid-cua-run-usage.js

@@ -2,16 +2,22 @@ const TOKENS_PER_MILLION = 1_000_000;
 const OPENAI_GPT_5_4_PRICING = Object.freeze({
     inputPerMillionUsd: 2.5,
 });
-const OPENAI_GPT_5_4_MINI_PRICING = Object.freeze({
-    inputPerMillionUsd: 0.75,
+const OPENAI_GPT_5_6_TERRA_PRICING = Object.freeze({
+    inputPerMillionUsd: 2.5,
+});
+const OPENAI_GPT_5_6_LUNA_PRICING = Object.freeze({
+    inputPerMillionUsd: 1,
 });
 const MODEL_PRICING = Object.freeze({
+    "gpt-5.6-terra": Object.freeze({
+        ...OPENAI_GPT_5_6_TERRA_PRICING,
+    }),
+    "gpt-5.6-luna": Object.freeze({
+        ...OPENAI_GPT_5_6_LUNA_PRICING,
+    }),
     "gpt-5.4": Object.freeze({
         ...OPENAI_GPT_5_4_PRICING,
     }),
-    "gpt-5.4-mini": Object.freeze({
-        ...OPENAI_GPT_5_4_MINI_PRICING,
-    }),
     // Loadmill Smart currently routes to GPT-5.4 behind the Loadmill LLM gateway.
     // Keep this explicit so pricing can be updated here if the backing model changes.
     "loadmill-smart": Object.freeze({
@@ -32,8 +38,11 @@ export function normalizeDroidCuaCostModel(model) {
     if (typeof model !== "string") {
         return null;
     }
-    if (model === "gpt-5.4-mini" || model.startsWith("gpt-5.4-mini-")) {
-        return "gpt-5.4-mini";
+    if (model === "gpt-5.6-terra" || model.startsWith("gpt-5.6-terra-")) {
+        return "gpt-5.6-terra";
+    }
+    if (model === "gpt-5.6-luna" || model.startsWith("gpt-5.6-luna-")) {
+        return "gpt-5.6-luna";
     }
     if (model === "gpt-5.4" || model.startsWith("gpt-5.4-")) {
         return "gpt-5.4";

package/build/src/core/execution-engine.js

@@ -1,7 +1,7 @@
 import { randomUUID } from "node:crypto";
 import { getScreenshotAsBase64, getCurrentPlatform } from "../device/connection.js";
 import { closeCurrentTab as closeCurrentBrowserTab, getBrowserInfo, navigateToUrl as navigateBrowserToUrl } from "../browser/connection.js";
-import { handleModelAction } from "../device/actions.js";
+import { closeKeyboard, handleModelAction } from "../device/actions.js";
 import { sendCUARequest } from "../device/openai.js";
 import { executeLoadmillInstruction } from "../device/loadmill.js";
 import { emitDesktopDebug } from "../utils/desktop-debug.js";
@@ -14,6 +14,7 @@ import { ScreenshotStore } from "./screenshot-store.js";
 import { handlePreviewClick } from "./tool-handlers/preview-click.js";
 import { handleNavigateToolCall } from "./tool-handlers/navigate.js";
 import { handleCloseCurrentTabToolCall } from "./tool-handlers/close-current-tab.js";
+import { handleCloseKeyboardToolCall } from "./tool-handlers/close-keyboard.js";
 import { handleAssertionToolCall } from "./tool-handlers/assert.js";
 import { handleLoadmillToolCall } from "./tool-handlers/loadmill.js";
 import { handleTestDesignCompleteToolCall } from "./tool-handlers/test-design-complete.js";
@@ -176,6 +177,8 @@ function formatActionForRecovery(action) {
         }
         case "scroll":
             return "Scrolled";
+        case "close_keyboard":
+            return "Closed the keyboard";
         default:
             return null;
     }
@@ -240,6 +243,7 @@ export class ExecutionEngine {
         this.stepDelayMs = options.stepDelayMs ?? getConfiguredStepDelayMs();
         this.getScreenshotAsBase64 = options.getScreenshotAsBase64 || getScreenshotAsBase64;
         this.handleModelAction = options.handleModelAction || handleModelAction;
+        this.closeKeyboard = options.closeKeyboard || closeKeyboard;
         this.sendCUARequest = options.sendCUARequest || sendCUARequest;
         this.reviseTestScript = options.reviseTestScript || null;
         this.getCurrentPlatform = options.getCurrentPlatform || getCurrentPlatform;
@@ -323,6 +327,8 @@ export class ExecutionEngine {
                 return await handleNavigateToolCall(handlerArgs);
             case "close_current_tab":
                 return await handleCloseCurrentTabToolCall(handlerArgs);
+            case "close_keyboard":
+                return await handleCloseKeyboardToolCall(handlerArgs);
             case "loadmill":
                 return await handleLoadmillToolCall(handlerArgs);
             case "type_sensitive_text":

package/build/src/core/runtime-config.js

@@ -90,11 +90,8 @@ export function inferLlmProviderModeFromCredentials(credentials = {}) {
 }
 export function resolveGatewayCuaModelAlias({ cuaModel } = {}) {
     const normalizedCuaModel = resolveCuaModel(cuaModel);
-    if (normalizedCuaModel === "loadmill-beacon") {
-        return LOADMILL_MODEL_ALIASES.BEACON;
-    }
-    return normalizedCuaModel === "gpt-5.4-mini"
-        ? LOADMILL_MODEL_ALIASES.FAST
+    return normalizedCuaModel === "loadmill-beacon"
+        ? LOADMILL_MODEL_ALIASES.BEACON
         : LOADMILL_MODEL_ALIASES.SMART;
 }
 export function resolveEffectiveCuaModel({ llmProviderMode = null, cuaModel = null, } = {}) {

package/build/src/core/session.js

@@ -45,7 +45,7 @@ export class Session {
         this.messages = [{ role: "system", content: prompt }];
     }
     /**
-     * Add a line to the transcript (for error recovery and logging)
+     * Add a line to the scenario transcript for assertions, recovery, and logging.
      */
     addToTranscript(line) {
         this.transcript.push(line);
@@ -57,7 +57,7 @@ export class Session {
         return this.transcript.join("\n");
     }
     /**
-     * Clear the per-instruction transcript.
+     * Clear the scenario transcript at a run boundary.
      */
     clearTranscript() {
         this.transcript = [];

package/build/src/core/tool-handlers/close-keyboard.js

@@ -0,0 +1,103 @@
+import { CAPTURE_SOURCES } from "../capture-sources.js";
+import { KEYBOARD_CLOSE_RESULTS, buildKeyboardCloseResult } from "../../device/keyboard.js";
+import { printCliOutput } from "../../utils/console-output.js";
+import { emitDesktopDebug } from "../../utils/desktop-debug.js";
+import { buildFunctionCallOutputItem, buildToolOutputContent, buildTextToolOutputItem, isDesignContext, parseJsonArguments, } from "./utils.js";
+function parseDismissalPoint(argumentsText, deviceInfo = {}) {
+    const parsed = parseJsonArguments(argumentsText || "{}");
+    if (!parsed.ok || !parsed.value || typeof parsed.value !== "object" || Array.isArray(parsed.value)) {
+        return { ok: false, message: "Invalid close_keyboard arguments: expected a JSON object." };
+    }
+    const hasX = Object.prototype.hasOwnProperty.call(parsed.value, "x");
+    const hasY = Object.prototype.hasOwnProperty.call(parsed.value, "y");
+    if (!hasX && !hasY) {
+        return { ok: true, value: null };
+    }
+    if (!hasX || !hasY || !Number.isFinite(parsed.value.x) || !Number.isFinite(parsed.value.y)) {
+        return { ok: false, message: "Invalid close_keyboard arguments: x and y must both be finite numbers." };
+    }
+    const point = { x: parsed.value.x, y: parsed.value.y };
+    const width = Number(deviceInfo.scaled_width);
+    const height = Number(deviceInfo.scaled_height);
+    // We can validate coordinate shape and bounds, but only the model can judge
+    // whether a cloud-iOS point is a safe dismissal target in the current UI.
+    if (point.x < 0 || point.y < 0 ||
+        (Number.isFinite(width) && point.x >= width) ||
+        (Number.isFinite(height) && point.y >= height)) {
+        return { ok: false, message: "Invalid close_keyboard arguments: the dismissal point is outside the current screenshot." };
+    }
+    return { ok: true, value: point };
+}
+export async function handleCloseKeyboardToolCall({ engine, callId, argumentsText, context = null, stepContext = null }) {
+    const session = engine.session;
+    if (session?.targetKind === "browser" || session?.deviceInfo?.platform === "browser") {
+        return {
+            kind: "continue",
+            outputItem: buildFunctionCallOutputItem(callId, buildToolOutputContent({ text: "close_keyboard is only supported for mobile runs." })),
+        };
+    }
+    const parsedPoint = parseDismissalPoint(argumentsText, session.deviceInfo);
+    if (!parsedPoint.ok) {
+        return {
+            kind: "continue",
+            outputItem: buildTextToolOutputItem(callId, parsedPoint.message),
+        };
+    }
+    const dismissalPoint = parsedPoint.value;
+    let result;
+    try {
+        result = await engine.closeKeyboard(session.deviceId, {
+            dismissalPoint,
+            scale: session.deviceInfo?.scale || 1,
+        });
+    }
+    catch (error) {
+        result = buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED, error);
+    }
+    const addOutput = context?.addOutput || printCliOutput;
+    const outputText = result.status === KEYBOARD_CLOSE_RESULTS.FAILED
+        ? `${result.message}. The preceding text entry was unaffected.`
+        : result.message;
+    const scope = isDesignContext(context) ? "design" : "execution";
+    const ids = {
+        ...(scope === "design" ? { sessionId: context?.sessionId } : { runId: context?.runId }),
+        ...(stepContext?.stepId ? { stepId: stepContext.stepId } : {}),
+        ...(Number.isInteger(stepContext?.instructionIndex)
+            ? { instructionIndex: stepContext.instructionIndex }
+            : {}),
+    };
+    const payload = {
+        platform: session.deviceInfo?.platform,
+        toolName: "close_keyboard",
+        callId,
+        status: result.status,
+        failed: result.status === KEYBOARD_CLOSE_RESULTS.FAILED,
+        ...(dismissalPoint ? { dismissalPoint } : {}),
+        ...(result.error ? { error: result.error } : {}),
+    };
+    emitDesktopDebug("tool_call", scope, ids, payload);
+    addOutput({
+        type: "action",
+        text: result.message,
+        eventType: "tool_call",
+        runId: context?.runId,
+        stepId: stepContext?.stepId,
+        instructionIndex: stepContext?.instructionIndex,
+        payload,
+    });
+    session.addToTranscript(`[Close keyboard] ${outputText}`);
+    const screenshotBase64 = await engine.getScreenshotAsBase64(session.deviceId, session.deviceInfo);
+    const currentScreenshot = await engine.recordScreenshot(screenshotBase64, {
+        runId: context?.runId,
+        sessionId: context?.sessionId,
+        stepId: stepContext?.stepId,
+        instructionIndex: stepContext?.instructionIndex,
+        captureSource: CAPTURE_SOURCES.CALL_OUTPUT_POST_ACTION,
+    });
+    return {
+        kind: "continue",
+        outputItem: buildFunctionCallOutputItem(callId, buildToolOutputContent({ text: outputText, imageBase64: screenshotBase64 })),
+        currentScreenshot,
+        executedActions: [{ type: "close_keyboard", ...(dismissalPoint || {}) }],
+    };
+}

package/build/src/device/actions.js

@@ -20,3 +20,10 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
     const backend = getDeviceBackend(platform);
     return backend.handleModelAction(deviceId, action, scale, context);
 }
+export async function closeKeyboard(deviceId, options = {}) {
+    const platform = getCurrentPlatform();
+    if (!platform) {
+        throw new Error("No platform set. Call connectToDevice first.");
+    }
+    return getDeviceBackend(platform).closeKeyboard(deviceId, options);
+}