@chatroomcp/chatroom 0.1.7 → 0.2.1

package/dist/plugins/computer/mcp.js

@@ -0,0 +1,126 @@
+import { z } from "zod";
+import { closedRead, openWorldMutation, } from "../../mcp/server/tool-support.js";
+import { currentMcpAccessScope } from "../../mcp/server/request-context.js";
+import { auditActionOutput, auditComputerActions, auditSnapshotOutput, } from "./audit.js";
+import { computerActionMetadataSchema, computerActionSchema, computerSnapshotMetadataSchema, computerSnapshotTargetSchema, } from "./computer-schemas.js";
+export function registerComputerTools(mcp, service) {
+    mcp.registerTool("computer_snapshot", {
+        title: "Observe computer",
+        description: "Observe the current desktop using a screenshot and compact accessibility elements. Element IDs are valid only for the returned snapshotId.",
+        inputSchema: z.object({
+            target: computerSnapshotTargetSchema.optional(),
+            includeScreenshot: z.boolean().default(true),
+            includeElements: z.boolean().default(true),
+        }),
+        outputSchema: computerSnapshotMetadataSchema,
+        annotations: closedRead,
+        action: "snapshot",
+        audit: {
+            output: (value) => auditSnapshotOutput(value),
+        },
+        present: presentSnapshot,
+    }, (input) => service.snapshot(currentMcpAccessScope(), {
+        includeScreenshot: input.includeScreenshot,
+        includeElements: input.includeElements,
+        ...(input.target === undefined
+            ? {}
+            : { target: normalizeSnapshotTarget(input.target) }),
+    }));
+    mcp.registerTool("computer_action", {
+        title: "Control computer",
+        description: "Execute a bounded batch of semantic or coordinate desktop actions. Prefer elementId from the latest snapshot; elementId actions require that snapshotId.",
+        inputSchema: z.object({
+            snapshotId: z.string().optional(),
+            actions: z.array(computerActionSchema).min(1).max(50),
+            observeAfter: z.boolean().default(true),
+        }),
+        outputSchema: computerActionMetadataSchema,
+        annotations: openWorldMutation,
+        action: "action",
+        audit: {
+            input: (value) => auditComputerActions(value),
+            output: (value) => auditActionOutput(value),
+        },
+        present: presentAction,
+    }, (input) => service.action(currentMcpAccessScope(), {
+        actions: input.actions,
+        observeAfter: input.observeAfter,
+        ...(input.snapshotId === undefined
+            ? {}
+            : { snapshotId: input.snapshotId }),
+    }));
+}
+function presentSnapshot(value) {
+    const snapshot = value;
+    const { screenshot, ...metadata } = snapshot;
+    return {
+        content: [
+            { type: "text", text: JSON.stringify(metadata, null, 2) },
+            ...(screenshot
+                ? [
+                    {
+                        type: "image",
+                        data: screenshot.data,
+                        mimeType: screenshot.mimeType,
+                    },
+                ]
+                : []),
+        ],
+        structuredContent: metadata,
+    };
+}
+function presentAction(value) {
+    const result = value;
+    const screenshot = result.snapshot?.screenshot;
+    const snapshot = result.snapshot
+        ? (({ screenshot: _screenshot, ...metadata }) => metadata)(result.snapshot)
+        : undefined;
+    const safe = {
+        success: result.success,
+        revision: result.revision,
+        executionMode: result.executionMode,
+        focusChanged: result.focusChanged,
+        ...(snapshot === undefined ? {} : { snapshot }),
+    };
+    return {
+        content: [
+            { type: "text", text: JSON.stringify(safe, null, 2) },
+            ...(screenshot
+                ? [
+                    {
+                        type: "image",
+                        data: screenshot.data,
+                        mimeType: screenshot.mimeType,
+                    },
+                ]
+                : []),
+        ],
+        structuredContent: safe,
+    };
+}
+function normalizeSnapshotTarget(input) {
+    switch (input.type) {
+        case "desktop":
+            return { type: "desktop" };
+        case "display":
+            return input.displayId === undefined
+                ? { type: "display" }
+                : { type: "display", displayId: input.displayId };
+        case "app":
+            return { type: "app", app: input.app };
+        case "window":
+            return { type: "window", elementId: input.elementId };
+        case "region": {
+            const region = {
+                type: "region",
+                x: input.x,
+                y: input.y,
+                width: input.width,
+                height: input.height,
+            };
+            return input.displayId === undefined
+                ? region
+                : { ...region, displayId: input.displayId };
+        }
+    }
+}

package/dist/plugins/computer/plugin.d.ts

@@ -0,0 +1,4 @@
+import type { InternalPlugin } from "../types.js";
+import { ComputerService } from "./computer-service.js";
+export declare const ComputerServiceToken: import("../types.js").ServiceToken<ComputerService>;
+export declare function createComputerPlugin(): InternalPlugin;

package/dist/plugins/computer/plugin.js

@@ -0,0 +1,29 @@
+import { createServiceToken } from "../types.js";
+import { NativeComputerBackend } from "./computer-native-backend.js";
+import { ComputerService } from "./computer-service.js";
+import { ComputerSettingsRepository } from "./computer-settings-repository.js";
+import { registerComputerTools } from "./mcp.js";
+import { currentMcpAccessScope } from "../../mcp/server/request-context.js";
+export const ComputerServiceToken = createServiceToken("computer");
+export function createComputerPlugin() {
+    let service = null;
+    return {
+        id: "computer",
+        activate(context) {
+            service = new ComputerService(new NativeComputerBackend(), new ComputerSettingsRepository(context.database), context.events);
+            context.services.provide(ComputerServiceToken, service);
+        },
+        registerMcp(mcp) {
+            if (!service)
+                throw new Error("Computer plugin is not active");
+            if (currentMcpAccessScope() === "remote" &&
+                !service.settings().remoteAccess)
+                return;
+            registerComputerTools(mcp, service);
+        },
+        async deactivate() {
+            await service?.shutdown();
+            service = null;
+        },
+    };
+}

package/dist/plugins/computer/types.d.ts

@@ -0,0 +1,174 @@
+export type ComputerAccessScope = "local" | "remote";
+export type ComputerPlatform = "macos" | "windows" | "linux" | "unsupported";
+export type ComputerHelperState = "running" | "stopped" | "unavailable";
+export type ComputerPermissionState = "granted" | "denied" | "unknown" | "not-required";
+export type ComputerPermission = "accessibility" | "screenRecording";
+export interface ComputerSettings {
+    enabled: boolean;
+    remoteAccess: boolean;
+    updatedAt: string;
+}
+export interface ComputerPermissions {
+    accessibility: ComputerPermissionState;
+    screenRecording: ComputerPermissionState;
+}
+export interface ComputerDisplay {
+    id: string;
+    name: string;
+    width: number;
+    height: number;
+    scale: number;
+    primary: boolean;
+}
+export interface ComputerStatus {
+    platform: ComputerPlatform;
+    helper: ComputerHelperState;
+    permissions: ComputerPermissions;
+    displays: ComputerDisplay[];
+    settings: ComputerSettings;
+}
+export interface ComputerElement {
+    id: number;
+    role: string;
+    name: string | null;
+    value: string | null;
+    enabled: boolean;
+    focused: boolean;
+    selected: boolean;
+    sensitive: boolean;
+    bounds: [number, number, number, number] | null;
+    actions: string[];
+}
+export type ComputerSnapshotTarget = {
+    type: "desktop";
+} | {
+    type: "display";
+    displayId?: string;
+} | {
+    type: "app";
+    app: string;
+} | {
+    type: "window";
+    elementId: number;
+} | {
+    type: "region";
+    displayId?: string;
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+};
+export interface ComputerSnapshotRequest {
+    target?: ComputerSnapshotTarget;
+    includeScreenshot: boolean;
+    includeElements: boolean;
+}
+export interface ComputerSnapshot {
+    snapshotId: string;
+    revision: number;
+    display: ComputerDisplay | null;
+    activeApp: string | null;
+    activeWindow: string | null;
+    cursor: {
+        x: number;
+        y: number;
+    } | null;
+    elements: ComputerElement[];
+    screenshot?: {
+        mimeType: "image/jpeg" | "image/png";
+        data: string;
+    };
+}
+export type ComputerAction = {
+    type: "move";
+    x: number;
+    y: number;
+} | {
+    type: "click";
+    x?: number;
+    y?: number;
+    elementId?: number;
+} | {
+    type: "double_click";
+    x?: number;
+    y?: number;
+    elementId?: number;
+} | {
+    type: "right_click";
+    x?: number;
+    y?: number;
+    elementId?: number;
+} | {
+    type: "drag";
+    from: {
+        x: number;
+        y: number;
+    };
+    to: {
+        x: number;
+        y: number;
+    };
+    durationMs?: number;
+} | {
+    type: "scroll";
+    deltaX?: number;
+    deltaY: number;
+    elementId?: number;
+} | {
+    type: "keypress";
+    keys: string[];
+} | {
+    type: "type_text";
+    text: string;
+    elementId?: number;
+} | {
+    type: "invoke";
+    elementId: number;
+} | {
+    type: "set_value";
+    elementId: number;
+    value: string;
+} | {
+    type: "select_text";
+    elementId: number;
+    start: number;
+    length: number;
+} | {
+    type: "activate_app";
+    app: string;
+} | {
+    type: "activate_window";
+    elementId: number;
+} | {
+    type: "move_window";
+    elementId: number;
+    x: number;
+    y: number;
+} | {
+    type: "resize_window";
+    elementId: number;
+    width: number;
+    height: number;
+} | {
+    type: "wait";
+    ms: number;
+};
+export interface ComputerActionRequest {
+    snapshotId?: string;
+    actions: ComputerAction[];
+    observeAfter: boolean;
+}
+export interface ComputerActionResult {
+    success: true;
+    revision: number;
+    snapshot?: ComputerSnapshot;
+    executionMode: "semantic" | "background" | "foreground" | "mixed";
+    focusChanged: boolean;
+}
+export interface ComputerBackend {
+    status(): Promise<Omit<ComputerStatus, "settings">>;
+    requestPermission(permission: ComputerPermission): Promise<Omit<ComputerStatus, "settings">>;
+    snapshot(request: ComputerSnapshotRequest, revision: number): Promise<ComputerSnapshot>;
+    action(request: ComputerActionRequest, revision: number): Promise<ComputerActionResult>;
+    dispose(): Promise<void>;
+}

package/dist/plugins/computer/types.js

@@ -0,0 +1 @@
+export {};

package/dist/plugins/web/api-types.d.ts

@@ -1,9 +1,23 @@
 import type { Operation as DomainOperation } from "../../core/operations/types.js";
 import type { GitInfo } from "../workspace/domain/git.js";
 import type { ProcessSnapshot } from "../process/types.js";
+import type { ComputerDisplay, ComputerPermission, ComputerSnapshot, ComputerStatus } from "../computer/types.js";
 import type { Workspace } from "../workspace/domain/workspace.js";
 import type { WorktreeApplyPreview, WorktreeFileDiff } from "../workspace/domain/review.js";
-export type { ProcessSnapshot, WorktreeApplyPreview, WorktreeFileDiff };
+export type { ComputerDisplay, ComputerPermission, ComputerStatus, ProcessSnapshot, WorktreeApplyPreview, WorktreeFileDiff, };
+export interface ComputerPreviewView {
+    snapshotId: string;
+    revision: number;
+    display: ComputerDisplay | null;
+    activeApp: string | null;
+    activeWindow: string | null;
+    cursor: {
+        x: number;
+        y: number;
+    } | null;
+    elementCount: number;
+    screenshot: ComputerSnapshot["screenshot"] | null;
+}
 export type Operation = DomainOperation;
 export interface WorkspaceView extends Workspace {
     git?: GitInfo | null;

package/dist/plugins/web/http/api-router.js

@@ -5,6 +5,7 @@ import { asyncRoute, parseCookie, requireString, } from "../../../presentation/h
 import { createProcessApiRouter } from "./process-api-router.js";
 import { createWorkspaceApiRouter } from "./workspace-api-router.js";
 import { createCloudApiRouter } from "./cloud-api-router.js";
+import { createComputerApiRouter } from "./computer-api-router.js";
 const SESSION_COOKIE = "chatroom_session";
 export function createApiRouter(application, eventBus, auth, passkeys, ingress, cloud, runtimeStatus) {
     const router = Router();
@@ -97,6 +98,7 @@ export function createApiRouter(application, eventBus, auth, passkeys, ingress,
     });
     router.use(createWorkspaceApiRouter(application));
     router.use(createProcessApiRouter(application));
+    router.use(createComputerApiRouter(application.computer, application.operations, ingress));
     router.use(createCloudApiRouter(cloud, application.operations));
     router.get("/events", (req, res) => {
         res.status(200);

package/dist/plugins/web/http/computer-api-router.d.ts

@@ -0,0 +1,5 @@
+import { Router } from "express";
+import type { ComputerService } from "../../computer/computer-service.js";
+import type { OperationLog } from "../../../operations/operation-log.js";
+import type { IngressPolicy } from "../../../auth/ingress-policy.js";
+export declare function createComputerApiRouter(computer: ComputerService, operations: OperationLog, ingress: IngressPolicy): Router;

package/dist/plugins/web/http/computer-api-router.js

@@ -0,0 +1,68 @@
+import { Router } from "express";
+import { z } from "zod";
+import { asyncRoute } from "../../../presentation/http/http-utils.js";
+import { ChatRoomError } from "../../../core/errors/chatroom-error.js";
+const settingsPatchSchema = z
+    .object({
+    enabled: z.boolean().optional(),
+    remoteAccess: z.boolean().optional(),
+})
+    .strict();
+export function createComputerApiRouter(computer, operations, ingress) {
+    const router = Router();
+    router.get("/computer/status", asyncRoute(async (_req, res) => res.json(await computer.status())));
+    router.post("/computer/permissions/accessibility/request", asyncRoute(async (req, res) => {
+        assertLocalPermissionRequest(req);
+        res.json(await computer.requestPermission("accessibility"));
+    }));
+    router.post("/computer/permissions/screen-recording/request", asyncRoute(async (req, res) => {
+        assertLocalPermissionRequest(req);
+        res.json(await computer.requestPermission("screenRecording"));
+    }));
+    function assertLocalPermissionRequest(req) {
+        if (ingress.isExternalWeb(req))
+            throw new ChatRoomError("FORBIDDEN", "Computer permissions can be requested only from the local WebUI");
+    }
+    router.patch("/computer/settings", asyncRoute(async (req, res) => {
+        const parsed = settingsPatchSchema.parse(req.body);
+        const patch = {
+            ...(parsed.enabled === undefined ? {} : { enabled: parsed.enabled }),
+            ...(parsed.remoteAccess === undefined
+                ? {}
+                : { remoteAccess: parsed.remoteAccess }),
+        };
+        res.json(await operations.run({
+            pluginId: "computer",
+            source: "gui",
+            action: "settings.set",
+            input: patch,
+        }, async () => computer.setSettings(patch)));
+    }));
+    router.get("/computer/preview", (req, res) => {
+        const scope = ingress.isExternalWeb(req) ? "remote" : "local";
+        res.json(presentPreview(computer.latestSnapshot(scope)));
+    });
+    router.post("/computer/snapshot", asyncRoute(async (req, res) => {
+        const scope = ingress.isExternalWeb(req) ? "remote" : "local";
+        const value = await computer.snapshot(scope, {
+            includeScreenshot: true,
+            includeElements: true,
+        });
+        res.json(presentPreview(value));
+    }));
+    return router;
+}
+function presentPreview(value) {
+    if (!value)
+        return null;
+    return {
+        snapshotId: value.snapshotId,
+        revision: value.revision,
+        display: value.display,
+        activeApp: value.activeApp,
+        activeWindow: value.activeWindow,
+        cursor: value.cursor,
+        elementCount: value.elements.length,
+        screenshot: value.screenshot ?? null,
+    };
+}

package/dist/plugins/web/plugin.js

@@ -1,6 +1,7 @@
 import { createServiceToken } from "../types.js";
 import { WorkspaceService } from "../workspace/plugin.js";
 import { ProcessService } from "../process/plugin.js";
+import { ComputerServiceToken } from "../computer/plugin.js";
 import { WebRuntime } from "./runtime.js";
 export const WebServiceToken = createServiceToken("web");
 export function createWebPlugin() {
@@ -9,8 +10,9 @@ export function createWebPlugin() {
         activate(context) {
             const workspace = context.services.require(WorkspaceService);
             const processes = context.services.require(ProcessService);
+            const computer = context.services.require(ComputerServiceToken);
             context.services.provide(WebServiceToken, {
-                application: new WebRuntime(workspace.workspaces, context.operations, processes, workspace.git),
+                application: new WebRuntime(workspace.workspaces, context.operations, processes, workspace.git, computer),
             });
         },
     };

package/dist/plugins/web/runtime.d.ts

@@ -2,12 +2,14 @@ import type { OperationLog } from "../../operations/operation-log.js";
 import type { WorkspaceService } from "../workspace/workspace-service.js";
 import type { ProcessSupervisor } from "../process/process-supervisor.js";
 import type { GitService } from "../workspace/git/git-service.js";
+import type { ComputerService } from "../computer/computer-service.js";
 export declare class WebRuntime {
     readonly workspaces: WorkspaceService;
     readonly operations: OperationLog;
     readonly processes: ProcessSupervisor;
     readonly git: GitService;
-    constructor(workspaces: WorkspaceService, operations: OperationLog, processes: ProcessSupervisor, git: GitService);
+    readonly computer: ComputerService;
+    constructor(workspaces: WorkspaceService, operations: OperationLog, processes: ProcessSupervisor, git: GitService, computer: ComputerService);
     previewWorktreeApply(workspaceId: string): Promise<import("./api-types.js").WorktreeApplyPreview>;
     previewWorktreeFileDiff(workspaceId: string, filePath: string): Promise<import("./api-types.js").WorktreeFileDiff>;
     applyWorktree(workspaceId: string, paths?: string[]): Promise<{

package/dist/plugins/web/runtime.js

@@ -3,11 +3,13 @@ export class WebRuntime {
     operations;
     processes;
     git;
-    constructor(workspaces, operations, processes, git) {
+    computer;
+    constructor(workspaces, operations, processes, git, computer) {
         this.workspaces = workspaces;
         this.operations = operations;
         this.processes = processes;
         this.git = git;
+        this.computer = computer;
     }
     previewWorktreeApply(workspaceId) {
         return this.workspaces.previewWorktreeApply(workspaceId);