@cloudcli-ai/cloudcli 1.33.3 → 1.35.0

package/dist-server/server/modules/computer-use/computer-use.service.js

@@ -0,0 +1,759 @@
+import { randomBytes, randomUUID } from 'node:crypto';
+import { spawn } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { appConfigDb } from '../../modules/database/index.js';
+import { providerMcpService } from '../../modules/providers/index.js';
+import { getModuleDir } from '../../utils/runtime-paths.js';
+import { getRuntimeReadiness as getExecutorReadiness, } from '../../modules/computer-use/computer-executor.js';
+import { runRawComputerAction } from '../../modules/computer-use/actions/raw-action-dispatcher.js';
+import { desktopAgentRelay } from '../../modules/computer-use/desktop-agent-relay.service.js';
+import { computerSemanticsService } from '../../modules/computer-use/computer-semantics.service.js';
+import { semanticOperationNames } from '../../modules/computer-use/semantics/semantic-tool-dispatcher.js';
+const __dirname = getModuleDir(import.meta.url);
+const IS_PLATFORM = process.env.VITE_IS_PLATFORM === 'true';
+const MAX_SESSIONS_PER_OWNER = Number.parseInt(process.env.CLOUDCLI_COMPUTER_USE_MAX_SESSIONS_PER_OWNER || '1', 10);
+const SESSION_TTL_MS = Number.parseInt(process.env.CLOUDCLI_COMPUTER_USE_SESSION_TTL_MS || String(30 * 60 * 1000), 10);
+const STOPPED_SESSION_RETENTION_MS = Number.parseInt(process.env.CLOUDCLI_COMPUTER_USE_STOPPED_SESSION_RETENTION_MS || String(30 * 60 * 1000), 10);
+const MAX_STORED_SESSIONS = Number.parseInt(process.env.CLOUDCLI_COMPUTER_USE_MAX_STORED_SESSIONS || '100', 10);
+const COMPUTER_USE_SETTINGS_KEY = 'computer_use_settings';
+const COMPUTER_USE_MCP_TOKEN_KEY = 'computer_use_mcp_token';
+const sessions = new Map();
+let installPromise = null;
+let lastInstallMessage = null;
+const DEFAULT_SETTINGS = {
+    enabled: false,
+};
+const AGENT_OWNER_ID = 'agent';
+const MCP_SERVER_NAME = 'cloudcli-computer-use';
+const MCP_PROVIDERS = ['claude', 'codex', 'cursor', 'gemini', 'opencode'];
+function getRuntime() {
+    return IS_PLATFORM ? 'cloud' : 'local';
+}
+function readSettings() {
+    try {
+        const raw = appConfigDb.get(COMPUTER_USE_SETTINGS_KEY);
+        if (!raw) {
+            return DEFAULT_SETTINGS;
+        }
+        const parsed = JSON.parse(raw);
+        return {
+            enabled: parsed.enabled === true,
+        };
+    }
+    catch (error) {
+        console.warn('[Computer Use] Failed to read settings:', error?.message || error);
+        return DEFAULT_SETTINGS;
+    }
+}
+function writeSettings(settings) {
+    const normalized = {
+        enabled: settings.enabled === true,
+    };
+    appConfigDb.set(COMPUTER_USE_SETTINGS_KEY, JSON.stringify(normalized));
+    return normalized;
+}
+function getOrCreateMcpToken() {
+    const existing = appConfigDb.get(COMPUTER_USE_MCP_TOKEN_KEY);
+    if (existing) {
+        return existing;
+    }
+    const token = randomBytes(32).toString('hex');
+    appConfigDb.set(COMPUTER_USE_MCP_TOKEN_KEY, token);
+    return token;
+}
+function getSetupMessage(settings, readiness) {
+    if (!settings.enabled) {
+        return 'Computer Use is disabled in settings.';
+    }
+    if (getRuntime() === 'cloud') {
+        return 'Open CloudCLI Desktop on this computer, connect the same account, and enable Computer Use.';
+    }
+    if (!readiness.nutInstalled || !readiness.screenshotInstalled) {
+        return 'Install the desktop control runtime to capture the screen and drive the mouse and keyboard.';
+    }
+    return readiness.installMessage || 'Computer Use runtime is not ready.';
+}
+function getMcpCommand() {
+    const serverDir = path.resolve(__dirname, '..', '..');
+    const mcpScriptPath = path.join(serverDir, 'computer-use-mcp.js');
+    if (fs.existsSync(mcpScriptPath)) {
+        return {
+            command: process.execPath,
+            args: [mcpScriptPath],
+        };
+    }
+    return {
+        command: 'cloudcli',
+        args: ['computer-use-mcp'],
+    };
+}
+function getMcpApiUrl() {
+    const port = process.env.SERVER_PORT || process.env.PORT || '3001';
+    return `http://127.0.0.1:${port}/api/computer-use-mcp`;
+}
+function getRuntimeReadiness() {
+    const base = getExecutorReadiness();
+    return {
+        ...base,
+        installInProgress: Boolean(installPromise),
+        installMessage: lastInstallMessage,
+    };
+}
+function runCommand(command, args) {
+    return new Promise((resolve, reject) => {
+        const child = spawn(command, args, {
+            cwd: process.cwd(),
+            env: process.env,
+            shell: false,
+            stdio: ['ignore', 'pipe', 'pipe'],
+        });
+        const output = [];
+        child.stdout.on('data', (chunk) => output.push(String(chunk)));
+        child.stderr.on('data', (chunk) => output.push(String(chunk)));
+        child.on('error', reject);
+        child.on('close', (code) => {
+            if (code === 0) {
+                resolve();
+                return;
+            }
+            reject(new Error(output.join('').trim() || `${command} ${args.join(' ')} exited with code ${code}`));
+        });
+    });
+}
+function formatInstallError(error) {
+    const message = error instanceof Error ? error.message : String(error);
+    if (process.platform === 'linux' && /libxtst|x11|xtst|libpng|imagemagick|scrot/i.test(message)) {
+        return [
+            'Installing the desktop control runtime needs system packages.',
+            'On Debian/Ubuntu run: sudo apt-get install -y libxtst-dev libpng-dev imagemagick',
+            'then try again.',
+        ].join(' ');
+    }
+    return message || 'Failed to install the Computer Use runtime.';
+}
+function isPackagedElectronNodeRuntime() {
+    return process.env.ELECTRON_RUN_AS_NODE === '1' && Boolean(process.versions.electron);
+}
+async function installRuntime() {
+    if (installPromise) {
+        return installPromise;
+    }
+    const readiness = getExecutorReadiness();
+    if (readiness.nutInstalled && readiness.screenshotInstalled) {
+        lastInstallMessage = 'Computer Use runtime is available.';
+        return { success: true, message: lastInstallMessage };
+    }
+    if (isPackagedElectronNodeRuntime()) {
+        lastInstallMessage = 'Computer Use runtime was not bundled with this desktop build.';
+        return { success: false, message: lastInstallMessage };
+    }
+    const npmCommand = process.platform === 'win32' ? 'npm.cmd' : 'npm';
+    installPromise = (async () => {
+        try {
+            lastInstallMessage = 'Installing desktop control runtime…';
+            await runCommand(npmCommand, [
+                'install',
+                '--no-save',
+                '--no-package-lock',
+                '@nut-tree-fork/nut-js',
+                'screenshot-desktop',
+            ]);
+            lastInstallMessage = 'Computer Use runtime installed.';
+            return { success: true, message: lastInstallMessage };
+        }
+        catch (error) {
+            lastInstallMessage = formatInstallError(error);
+            return { success: false, message: lastInstallMessage };
+        }
+    })();
+    try {
+        return await installPromise;
+    }
+    finally {
+        installPromise = null;
+    }
+}
+function getOwnerId(owner) {
+    if (owner.id === undefined || owner.id === null || String(owner.id).trim() === '') {
+        throw new Error('Authenticated user is required.');
+    }
+    return String(owner.id);
+}
+function publicSession(session) {
+    const { ownerId: _ownerId, ...publicFields } = session;
+    return publicFields;
+}
+function ownerSessions(ownerId) {
+    return [...sessions.values()].filter((session) => session.ownerId === ownerId);
+}
+function canAccessSession(ownerId, session) {
+    return session.ownerId === ownerId || session.ownerId === AGENT_OWNER_ID;
+}
+function normalizeSessionId(sessionId) {
+    if (typeof sessionId !== 'string') {
+        return null;
+    }
+    const trimmed = sessionId.trim();
+    return trimmed ? trimmed : null;
+}
+function findActiveAgentSession() {
+    return ownerSessions(AGENT_OWNER_ID)
+        .filter((session) => session.status === 'ready')
+        .sort((a, b) => Date.parse(b.updatedAt) - Date.parse(a.updatedAt))[0] || null;
+}
+function positiveDuration(value, fallback) {
+    return Number.isFinite(value) && value > 0 ? value : fallback;
+}
+async function expireStaleSessions(now = Date.now()) {
+    const sessionTtl = positiveDuration(SESSION_TTL_MS, 30 * 60 * 1000);
+    const stoppedRetention = positiveDuration(STOPPED_SESSION_RETENTION_MS, sessionTtl);
+    for (const [sessionId, session] of sessions.entries()) {
+        const updatedAt = Date.parse(session.updatedAt);
+        if (!Number.isFinite(updatedAt)) {
+            continue;
+        }
+        if (session.status === 'ready') {
+            if (now - updatedAt <= sessionTtl) {
+                continue;
+            }
+            session.status = 'stopped';
+            session.agentAccessEnabled = false;
+            session.updatedAt = new Date(now).toISOString();
+            session.lastAction = 'expire';
+            session.message = 'Computer Use session expired after inactivity.';
+            continue;
+        }
+        if (now - updatedAt > stoppedRetention) {
+            sessions.delete(sessionId);
+        }
+    }
+    const maxStoredSessions = Number.isFinite(MAX_STORED_SESSIONS) && MAX_STORED_SESSIONS > 0
+        ? MAX_STORED_SESSIONS
+        : 100;
+    if (sessions.size <= maxStoredSessions) {
+        return;
+    }
+    const removable = [...sessions.values()]
+        .filter((session) => session.status !== 'ready')
+        .sort((a, b) => Date.parse(a.updatedAt) - Date.parse(b.updatedAt));
+    for (const session of removable) {
+        if (sessions.size <= maxStoredSessions) {
+            break;
+        }
+        sessions.delete(session.id);
+    }
+}
+function applyRelayResult(session, result) {
+    if (typeof result.screenshotDataUrl === 'string') {
+        session.screenshotDataUrl = result.screenshotDataUrl;
+    }
+    if (result.displaySize) {
+        session.displaySize = result.displaySize;
+    }
+    if (result.cursor) {
+        session.cursor = { x: result.cursor.x, y: result.cursor.y, actor: session.cursor?.actor ?? 'agent' };
+    }
+    session.updatedAt = new Date().toISOString();
+}
+function stripSessionArgs(args) {
+    const { sessionId: _sessionId, ...toolArgs } = args;
+    return toolArgs;
+}
+async function refreshScreenshot(session) {
+    if (getRuntime() === 'cloud') {
+        const result = (await desktopAgentRelay.relay('screenshot', { sessionId: session.id }));
+        applyRelayResult(session, result);
+        return;
+    }
+    applyRelayResult(session, await runRawComputerAction({ type: 'screenshot' }, session));
+}
+/** Runs one action and refreshes the session screenshot afterwards. */
+async function performAction(session, action) {
+    if (getRuntime() === 'cloud') {
+        const result = (await desktopAgentRelay.relay(action.type, {
+            ...action,
+            sessionId: session.id,
+            displaySize: session.displaySize,
+        }));
+        applyRelayResult(session, result);
+        return;
+    }
+    applyRelayResult(session, await runRawComputerAction(action, session));
+}
+/** Reads the current cursor position in screenshot-pixel space. */
+async function getCursorPosition(session) {
+    if (getRuntime() === 'cloud') {
+        const result = (await desktopAgentRelay.relay('cursor_position', {
+            sessionId: session.id,
+            displaySize: session.displaySize,
+        }));
+        applyRelayResult(session, result);
+        if (result.position) {
+            return result.position;
+        }
+        return session.cursor ? { x: session.cursor.x, y: session.cursor.y } : { x: 0, y: 0 };
+    }
+    const result = await runRawComputerAction({ type: 'cursor_position' }, session);
+    applyRelayResult(session, result);
+    return result.position || session.cursor || { x: 0, y: 0 };
+}
+function assertReady(session) {
+    if (session.status !== 'ready') {
+        throw new Error(session.message || 'Computer Use session is not available.');
+    }
+}
+function agentToolsAvailable() {
+    const settings = readSettings();
+    if (!settings.enabled) {
+        return false;
+    }
+    if (getRuntime() === 'cloud') {
+        return desktopAgentRelay.isConnected();
+    }
+    return true;
+}
+function assertAgentToolsAvailable() {
+    if (agentToolsAvailable()) {
+        return;
+    }
+    const settings = readSettings();
+    if (!settings.enabled) {
+        throw new Error('Computer Use agent tools are disabled.');
+    }
+    throw new Error(getRuntime() === 'cloud'
+        ? 'No desktop is linked. Open CloudCLI Desktop on this computer, connect the same account, and enable Computer Use.'
+        : 'Computer Use agent tools are disabled.');
+}
+function stopSessions(lastAction, message) {
+    for (const session of sessions.values()) {
+        session.status = 'stopped';
+        session.agentAccessEnabled = false;
+        session.updatedAt = new Date().toISOString();
+        session.lastAction = lastAction;
+        session.message = message;
+    }
+}
+export const computerUseService = {
+    async getSettings() {
+        return readSettings();
+    },
+    async updateSettings(settings) {
+        const current = readSettings();
+        const enabled = typeof settings.enabled === 'boolean' ? settings.enabled : current.enabled;
+        const next = writeSettings({ enabled });
+        if (next.enabled) {
+            await this.registerAgentMcp();
+        }
+        else {
+            await this.unregisterAgentMcp();
+            desktopAgentRelay.disconnectAll('Computer Use was disabled in this environment.');
+            stopSessions('settings:disabled', 'Computer Use was disabled in settings.');
+        }
+        return next;
+    },
+    async getStatus() {
+        const settings = readSettings();
+        const readiness = getRuntimeReadiness();
+        const isCloud = getRuntime() === 'cloud';
+        const runtimeReady = readiness.nutInstalled && readiness.screenshotInstalled;
+        // Cloud mode still respects the saved feature setting. When enabled, cloud
+        // availability comes from a linked desktop agent because the hosted server
+        // has no screen of its own.
+        const desktopAgentConnected = desktopAgentRelay.isConnected();
+        const available = settings.enabled && (isCloud
+            ? desktopAgentConnected
+            : runtimeReady);
+        return {
+            enabled: settings.enabled,
+            runtime: getRuntime(),
+            available,
+            desktopAgentConnected,
+            desktopAgentCount: desktopAgentRelay.connectedCount(),
+            nutInstalled: readiness.nutInstalled,
+            screenshotInstalled: readiness.screenshotInstalled,
+            installInProgress: readiness.installInProgress,
+            sessionCount: sessions.size,
+            message: available ? 'Computer Use runtime is available.' : getSetupMessage(settings, readiness),
+        };
+    },
+    async registerAgentMcp() {
+        const { command, args } = getMcpCommand();
+        const results = await providerMcpService.addMcpServerToAllProviders({
+            name: MCP_SERVER_NAME,
+            scope: 'user',
+            transport: 'stdio',
+            command,
+            args,
+            env: {
+                CLOUDCLI_COMPUTER_USE_MCP_TOKEN: getOrCreateMcpToken(),
+                CLOUDCLI_COMPUTER_USE_API_URL: getMcpApiUrl(),
+            },
+        });
+        return { name: MCP_SERVER_NAME, command, args, results };
+    },
+    getMcpToken() {
+        return getOrCreateMcpToken();
+    },
+    async unregisterAgentMcp() {
+        const results = await Promise.all(MCP_PROVIDERS.map(async (provider) => {
+            try {
+                const result = await providerMcpService.removeProviderMcpServer(provider, {
+                    name: MCP_SERVER_NAME,
+                    scope: 'user',
+                });
+                return { provider, removed: result.removed };
+            }
+            catch (error) {
+                return {
+                    provider,
+                    removed: false,
+                    error: error instanceof Error ? error.message : 'Unknown error',
+                };
+            }
+        }));
+        return { name: MCP_SERVER_NAME, results };
+    },
+    async installRuntime() {
+        const result = await installRuntime();
+        return {
+            ...result,
+            status: await this.getStatus(),
+        };
+    },
+    async listSessions(owner) {
+        const ownerId = getOwnerId(owner);
+        await expireStaleSessions();
+        return [...sessions.values()]
+            .filter((session) => canAccessSession(ownerId, session))
+            .map(publicSession);
+    },
+    async createSession(owner, options) {
+        const ownerId = getOwnerId(owner);
+        await expireStaleSessions();
+        const createdBy = options?.createdBy ?? 'user';
+        const now = new Date().toISOString();
+        const session = {
+            id: randomUUID(),
+            ownerId,
+            createdBy,
+            runtime: getRuntime(),
+            status: 'unavailable',
+            screenshotDataUrl: null,
+            createdAt: now,
+            updatedAt: now,
+            lastAction: 'create',
+            // Consent is always OFF at creation — the user must explicitly grant control,
+            // even for agent-initiated sessions controlling the full desktop.
+            agentAccessEnabled: false,
+            displaySize: null,
+            message: null,
+            cursor: null,
+        };
+        const activeOwnerSessions = ownerSessions(ownerId).filter((item) => item.status === 'ready');
+        if (activeOwnerSessions.length >= MAX_SESSIONS_PER_OWNER) {
+            throw new Error(`Computer Use is limited to ${MAX_SESSIONS_PER_OWNER} active session(s).`);
+        }
+        const settings = readSettings();
+        const readiness = getRuntimeReadiness();
+        const isCloud = getRuntime() === 'cloud';
+        const runtimeReady = readiness.nutInstalled && readiness.screenshotInstalled;
+        const ready = settings.enabled && (isCloud
+            ? desktopAgentRelay.isConnected()
+            : runtimeReady);
+        if (!ready) {
+            session.message = getSetupMessage(settings, readiness);
+            sessions.set(session.id, session);
+            return publicSession(session);
+        }
+        // In cloud mode the linked desktop agent is the consent authority and prompts
+        // the user per its own consent mode, so the relay is allowed to act. In local
+        // mode the user must still grant control from the panel.
+        if (isCloud) {
+            session.agentAccessEnabled = true;
+        }
+        session.status = 'ready';
+        session.message = isCloud
+            ? 'Computer Use session is ready on the linked desktop.'
+            : 'Computer Use session is ready. Grant control to let agents act.';
+        sessions.set(session.id, session);
+        try {
+            await refreshScreenshot(session);
+        }
+        catch (error) {
+            session.status = 'unavailable';
+            session.message = error instanceof Error ? error.message : 'Failed to capture the screen.';
+        }
+        return publicSession(session);
+    },
+    async grantAgentAccess(owner, sessionId) {
+        const ownerId = getOwnerId(owner);
+        const session = sessions.get(sessionId);
+        if (!session || !canAccessSession(ownerId, session)) {
+            throw new Error('Computer Use session not found.');
+        }
+        session.agentAccessEnabled = true;
+        session.updatedAt = new Date().toISOString();
+        session.lastAction = 'consent:grant';
+        return publicSession(session);
+    },
+    async revokeAgentAccess(owner, sessionId) {
+        const ownerId = getOwnerId(owner);
+        const session = sessions.get(sessionId);
+        if (!session || !canAccessSession(ownerId, session)) {
+            throw new Error('Computer Use session not found.');
+        }
+        session.agentAccessEnabled = false;
+        session.updatedAt = new Date().toISOString();
+        session.lastAction = 'consent:revoke';
+        return publicSession(session);
+    },
+    async stopSession(owner, sessionId) {
+        const ownerId = getOwnerId(owner);
+        const session = sessions.get(sessionId);
+        if (!session || !canAccessSession(ownerId, session)) {
+            return { stopped: false };
+        }
+        session.status = 'stopped';
+        session.agentAccessEnabled = false;
+        session.updatedAt = new Date().toISOString();
+        session.lastAction = 'stop';
+        session.message = 'Computer Use session stopped. Agent control is revoked.';
+        if (getRuntime() === 'cloud' && desktopAgentRelay.isConnected()) {
+            // Best-effort: tell the desktop agent to forget this session's consent.
+            void desktopAgentRelay.relay('stop_session', { sessionId }).catch(() => undefined);
+        }
+        return { stopped: true, session: publicSession(session) };
+    },
+    async deleteSession(owner, sessionId) {
+        const ownerId = getOwnerId(owner);
+        const session = sessions.get(sessionId);
+        if (!session || !canAccessSession(ownerId, session)) {
+            return { deleted: false };
+        }
+        sessions.delete(sessionId);
+        return { deleted: true, sessionId };
+    },
+    // --- User-initiated actions (from the panel) -------------------------------
+    async userScreenshot(owner, sessionId) {
+        const ownerId = getOwnerId(owner);
+        const session = sessions.get(sessionId);
+        if (!session || !canAccessSession(ownerId, session)) {
+            throw new Error('Computer Use session not found.');
+        }
+        assertReady(session);
+        await refreshScreenshot(session);
+        session.lastAction = 'screenshot';
+        return publicSession(session);
+    },
+    async userClick(owner, sessionId, input) {
+        const ownerId = getOwnerId(owner);
+        const session = sessions.get(sessionId);
+        if (!session || !canAccessSession(ownerId, session)) {
+            throw new Error('Computer Use session not found.');
+        }
+        assertReady(session);
+        await performAction(session, {
+            type: 'click',
+            button: input.button || 'left',
+            point: { x: input.x, y: input.y },
+            double: input.double === true,
+        });
+        session.cursor = { x: input.x, y: input.y, actor: 'user' };
+        session.lastAction = input.double ? 'double_click' : 'click';
+        return publicSession(session);
+    },
+    async userPressKey(owner, sessionId, key) {
+        const ownerId = getOwnerId(owner);
+        const session = sessions.get(sessionId);
+        if (!session || !canAccessSession(ownerId, session)) {
+            throw new Error('Computer Use session not found.');
+        }
+        assertReady(session);
+        await performAction(session, { type: 'key', key });
+        session.lastAction = `key:${key}`;
+        return publicSession(session);
+    },
+    // --- Agent-initiated actions (via MCP) ------------------------------------
+    /**
+     * Resolves a session the agent is allowed to act on. In local mode this
+     * enforces the in-process per-session consent flag. In cloud mode the linked
+     * desktop agent is the consent authority (it prompts the user per its own
+     * consent mode), so this only requires the relay to be connected.
+     */
+    async getOrCreateAgentSession() {
+        assertAgentToolsAvailable();
+        await expireStaleSessions();
+        const existing = findActiveAgentSession();
+        if (existing) {
+            return existing;
+        }
+        const created = await this.createSession({ id: AGENT_OWNER_ID }, { createdBy: 'agent' });
+        const session = sessions.get(created.id);
+        if (!session) {
+            throw new Error('Computer Use session could not be created.');
+        }
+        return session;
+    },
+    async getConsentedSession(sessionId) {
+        assertAgentToolsAvailable();
+        const normalizedSessionId = normalizeSessionId(sessionId);
+        const session = normalizedSessionId
+            ? sessions.get(normalizedSessionId)
+            : await this.getOrCreateAgentSession();
+        if (!session) {
+            throw new Error('Computer Use session not found.');
+        }
+        if (getRuntime() !== 'cloud' && !session.agentAccessEnabled) {
+            throw new Error(`Computer Use session ${session.id} is awaiting user consent. Ask the user to grant control in the Computer panel.`);
+        }
+        assertReady(session);
+        return session;
+    },
+    async agentScreenshot(sessionId) {
+        const session = await this.getConsentedSession(sessionId);
+        await refreshScreenshot(session);
+        session.lastAction = 'screenshot';
+        return publicSession(session);
+    },
+    async agentCursorPosition(sessionId) {
+        const session = await this.getConsentedSession(sessionId);
+        const point = await getCursorPosition(session);
+        session.cursor = { ...point, actor: 'agent' };
+        session.lastAction = 'cursor_position';
+        return { session: publicSession(session), position: point };
+    },
+    async agentMouseMove(sessionId, point) {
+        const session = await this.getConsentedSession(sessionId);
+        await performAction(session, { type: 'mouse_move', point });
+        session.cursor = { ...point, actor: 'agent' };
+        session.lastAction = 'mouse_move';
+        return publicSession(session);
+    },
+    async agentUnifiedClick(sessionId, input) {
+        const session = await this.getConsentedSession(sessionId);
+        const button = input.button || 'left';
+        const clickCount = Math.max(1, Math.min(Math.trunc(input.clickCount || 1), 5));
+        for (let index = 0; index < clickCount; index += 1) {
+            await performAction(session, { type: 'click', button, point: input.point, double: false });
+        }
+        if (input.point) {
+            session.cursor = { ...input.point, actor: 'agent' };
+        }
+        session.lastAction = clickCount > 1 ? `${button}_click:${clickCount}` : `${button}_click`;
+        return publicSession(session);
+    },
+    async agentDrag(sessionId, from, to, button = 'left') {
+        const session = await this.getConsentedSession(sessionId);
+        await performAction(session, { type: 'drag', from, to, button });
+        session.cursor = { ...to, actor: 'agent' };
+        session.lastAction = `${button}_drag`;
+        return publicSession(session);
+    },
+    async agentType(sessionId, text) {
+        const session = await this.getConsentedSession(sessionId);
+        await performAction(session, { type: 'type', text });
+        session.lastAction = 'type';
+        return publicSession(session);
+    },
+    async agentKey(sessionId, key) {
+        const session = await this.getConsentedSession(sessionId);
+        await performAction(session, { type: 'key', key });
+        session.lastAction = `key:${key}`;
+        return publicSession(session);
+    },
+    async agentScroll(sessionId, input) {
+        const session = await this.getConsentedSession(sessionId);
+        const point = typeof input.x === 'number' && typeof input.y === 'number' ? { x: input.x, y: input.y } : undefined;
+        await performAction(session, { type: 'scroll', direction: input.direction, amount: input.amount, point });
+        if (point) {
+            session.cursor = { ...point, actor: 'agent' };
+        }
+        session.lastAction = `scroll:${input.direction}`;
+        return publicSession(session);
+    },
+    async agentWait(sessionId, timeoutMs) {
+        const session = await this.getConsentedSession(sessionId);
+        await performAction(session, { type: 'wait', ms: timeoutMs });
+        session.lastAction = 'wait';
+        return publicSession(session);
+    },
+    async agentStopSession(sessionId) {
+        assertAgentToolsAvailable();
+        const normalizedSessionId = normalizeSessionId(sessionId);
+        if (normalizedSessionId) {
+            return this.stopSession({ id: AGENT_OWNER_ID }, normalizedSessionId);
+        }
+        await expireStaleSessions();
+        const existing = findActiveAgentSession();
+        if (!existing) {
+            return { stopped: false };
+        }
+        return this.stopSession({ id: AGENT_OWNER_ID }, existing.id);
+    },
+    async callSemanticTool(toolName, args) {
+        if (!semanticOperationNames.has(toolName)) {
+            throw new Error(`Unsupported semantic Computer Use tool: ${toolName}`);
+        }
+        const sessionId = typeof args.sessionId === 'string' ? args.sessionId : undefined;
+        const session = await this.getConsentedSession(normalizeSessionId(sessionId) ?? undefined);
+        const toolArgs = { ...stripSessionArgs(args), sessionId: session.id };
+        const semanticResult = getRuntime() === 'cloud'
+            ? await desktopAgentRelay.relay('semantic_tool', {
+                sessionId: session.id,
+                displaySize: session.displaySize,
+                toolName,
+                arguments: toolArgs,
+            })
+            : await computerSemanticsService.callTool(toolName, toolArgs);
+        applyRelayResult(session, semanticResult);
+        session.lastAction = `semantic:${toolName}`;
+        return { session: publicSession(session), result: semanticResult };
+    },
+    /**
+     * Cloud only: when a desktop agent links to this hosted environment, expose
+     * the computer_* MCP tools only if the user enabled Computer Use in settings.
+     */
+    async onDesktopAgentConnected() {
+        if (getRuntime() !== 'cloud') {
+            return;
+        }
+        if (!readSettings().enabled) {
+            return;
+        }
+        try {
+            await this.registerAgentMcp();
+        }
+        catch (error) {
+            console.warn('[Computer Use] Failed to register MCP for linked desktop agent:', error instanceof Error ? error.message : error);
+        }
+    },
+    /** Cloud only: tear down sessions when the last desktop agent disconnects. */
+    async onDesktopAgentDisconnected() {
+        if (getRuntime() !== 'cloud' || desktopAgentRelay.isConnected()) {
+            return;
+        }
+        for (const session of sessions.values()) {
+            if (session.status === 'ready') {
+                session.status = 'stopped';
+                session.agentAccessEnabled = false;
+                session.updatedAt = new Date().toISOString();
+                session.lastAction = 'agent-disconnected';
+                session.message = 'The linked desktop agent disconnected.';
+            }
+        }
+    },
+    async stopAllSessions() {
+        stopSessions('shutdown', 'Computer Use session stopped during server shutdown.');
+    },
+};
+// Drive cloud MCP exposure + session teardown off desktop-agent connectivity.
+desktopAgentRelay.setHooks({
+    canAcceptConnection: () => getRuntime() === 'cloud' && readSettings().enabled,
+    onFirstConnect: () => computerUseService.onDesktopAgentConnected(),
+    onLastDisconnect: () => computerUseService.onDesktopAgentDisconnected(),
+});
+process.once('beforeExit', () => {
+    void computerUseService.stopAllSessions();
+});
+//# sourceMappingURL=computer-use.service.js.map