@chatroomcp/chatroom 0.1.7 → 0.2.0

package/dist/plugins/computer/computer-schemas.js

@@ -0,0 +1,152 @@
+import { z } from "zod";
+export const computerPermissionStateSchema = z.enum([
+    "granted",
+    "denied",
+    "unknown",
+    "not-required",
+]);
+export const computerDisplaySchema = z.object({
+    id: z.string(),
+    name: z.string(),
+    width: z.number(),
+    height: z.number(),
+    scale: z.number(),
+    primary: z.boolean(),
+});
+export const computerElementSchema = z.object({
+    id: z.number().int(),
+    role: z.string(),
+    name: z.string().nullable(),
+    value: z.string().nullable(),
+    enabled: z.boolean(),
+    focused: z.boolean(),
+    selected: z.boolean(),
+    sensitive: z.boolean(),
+    bounds: z.tuple([z.number(), z.number(), z.number(), z.number()]).nullable(),
+    actions: z.array(z.string()),
+});
+export const computerScreenshotSchema = z.object({
+    mimeType: z.enum(["image/jpeg", "image/png"]),
+    data: z.string(),
+});
+export const computerNativeStatusSchema = z.object({
+    platform: z.enum(["macos", "windows", "unsupported"]),
+    helper: z.enum(["running", "stopped", "unavailable"]),
+    permissions: z.object({
+        accessibility: computerPermissionStateSchema,
+        screenRecording: computerPermissionStateSchema,
+    }),
+    displays: z.array(computerDisplaySchema),
+});
+export const computerSnapshotSchema = z.object({
+    snapshotId: z.string().min(1),
+    revision: z.number().int(),
+    display: computerDisplaySchema.nullable(),
+    activeApp: z.string().nullable(),
+    activeWindow: z.string().nullable(),
+    cursor: z.object({ x: z.number(), y: z.number() }).nullable(),
+    elements: z.array(computerElementSchema),
+    screenshot: computerScreenshotSchema.optional(),
+});
+export const computerSnapshotMetadataSchema = computerSnapshotSchema.omit({
+    screenshot: true,
+});
+export const computerActionSchema = z.discriminatedUnion("type", [
+    z.object({ type: z.literal("move"), x: z.number(), y: z.number() }),
+    z.object({
+        type: z.literal("click"),
+        x: z.number().optional(),
+        y: z.number().optional(),
+        elementId: z.number().int().optional(),
+    }),
+    z.object({
+        type: z.literal("double_click"),
+        x: z.number().optional(),
+        y: z.number().optional(),
+        elementId: z.number().int().optional(),
+    }),
+    z.object({
+        type: z.literal("right_click"),
+        x: z.number().optional(),
+        y: z.number().optional(),
+        elementId: z.number().int().optional(),
+    }),
+    z.object({
+        type: z.literal("drag"),
+        from: z.object({ x: z.number(), y: z.number() }),
+        to: z.object({ x: z.number(), y: z.number() }),
+        durationMs: z.number().int().min(0).max(10_000).optional(),
+    }),
+    z.object({
+        type: z.literal("scroll"),
+        deltaX: z.number().optional(),
+        deltaY: z.number(),
+        elementId: z.number().int().optional(),
+    }),
+    z.object({
+        type: z.literal("keypress"),
+        keys: z.array(z.string().min(1)).min(1).max(8),
+    }),
+    z.object({
+        type: z.literal("type_text"),
+        text: z.string().max(100_000),
+        elementId: z.number().int().optional(),
+    }),
+    z.object({ type: z.literal("invoke"), elementId: z.number().int() }),
+    z.object({
+        type: z.literal("set_value"),
+        elementId: z.number().int(),
+        value: z.string().max(100_000),
+    }),
+    z.object({
+        type: z.literal("select_text"),
+        elementId: z.number().int(),
+        start: z.number().int().min(0),
+        length: z.number().int().min(0),
+    }),
+    z.object({ type: z.literal("activate_app"), app: z.string().min(1) }),
+    z.object({ type: z.literal("activate_window"), elementId: z.number().int() }),
+    z.object({
+        type: z.literal("move_window"),
+        elementId: z.number().int(),
+        x: z.number(),
+        y: z.number(),
+    }),
+    z.object({
+        type: z.literal("resize_window"),
+        elementId: z.number().int(),
+        width: z.number().positive(),
+        height: z.number().positive(),
+    }),
+    z.object({
+        type: z.literal("wait"),
+        ms: z.number().int().min(0).max(30_000),
+    }),
+]);
+export const computerActionResultSchema = z.object({
+    success: z.literal(true),
+    revision: z.number().int(),
+    snapshot: computerSnapshotSchema.optional(),
+    executionMode: z.enum(["semantic", "background", "foreground", "mixed"]),
+    focusChanged: z.boolean(),
+});
+export const computerActionMetadataSchema = computerActionResultSchema.extend({
+    snapshot: computerSnapshotMetadataSchema.optional(),
+});
+export const computerSnapshotTargetSchema = z.discriminatedUnion("type", [
+    z.object({ type: z.literal("desktop") }),
+    z.object({
+        type: z.literal("display"),
+        displayId: z.string().optional(),
+    }),
+    z.object({ type: z.literal("app"), app: z.string().min(1) }),
+    z.object({ type: z.literal("window"), elementId: z.number().int() }),
+    z.object({
+        type: z.literal("region"),
+        displayId: z.string().optional(),
+        x: z.number(),
+        y: z.number(),
+        width: z.number().positive(),
+        height: z.number().positive(),
+    }),
+]);

package/dist/plugins/computer/computer-service.d.ts

@@ -0,0 +1,27 @@
+import type { RuntimeEventBus } from "../../app/event-bus.js";
+import type { ComputerSettingsRepository } from "./computer-settings-repository.js";
+import type { ComputerAccessScope, ComputerActionRequest, ComputerActionResult, ComputerBackend, ComputerPermission, ComputerSettings, ComputerSnapshot, ComputerSnapshotRequest, ComputerStatus } from "./types.js";
+type ComputerSettingsStore = Pick<ComputerSettingsRepository, "get" | "set">;
+export declare class ComputerService {
+    private readonly backend;
+    private readonly settingsRepository;
+    private readonly events;
+    private revision;
+    private currentSnapshotId;
+    private latestSnapshotValue;
+    private queue;
+    constructor(backend: ComputerBackend, settingsRepository: ComputerSettingsStore, events: RuntimeEventBus);
+    settings(): ComputerSettings;
+    latestSnapshot(scope: ComputerAccessScope): ComputerSnapshot | null;
+    requestPermission(permission: ComputerPermission): Promise<ComputerStatus>;
+    status(): Promise<ComputerStatus>;
+    setSettings(patch: Partial<Pick<ComputerSettings, "enabled" | "remoteAccess">>): ComputerSettings;
+    snapshot(scope: ComputerAccessScope, request: ComputerSnapshotRequest): Promise<ComputerSnapshot>;
+    action(scope: ComputerAccessScope, request: ComputerActionRequest): Promise<ComputerActionResult>;
+    shutdown(): Promise<void>;
+    private assertActionBudget;
+    private assertAllowed;
+    private assertScopeAllowed;
+    private serial;
+}
+export {};

package/dist/plugins/computer/computer-service.js

@@ -0,0 +1,108 @@
+import { randomUUID } from "node:crypto";
+import { ChatRoomError } from "../../core/errors/chatroom-error.js";
+export class ComputerService {
+    backend;
+    settingsRepository;
+    events;
+    revision = 0;
+    currentSnapshotId = null;
+    latestSnapshotValue = null;
+    queue = Promise.resolve();
+    constructor(backend, settingsRepository, events) {
+        this.backend = backend;
+        this.settingsRepository = settingsRepository;
+        this.events = events;
+    }
+    settings() {
+        return this.settingsRepository.get();
+    }
+    latestSnapshot(scope) {
+        const settings = this.settings();
+        if (!settings.enabled)
+            return null;
+        this.assertScopeAllowed(scope, settings);
+        return this.latestSnapshotValue;
+    }
+    requestPermission(permission) {
+        return this.serial(async () => {
+            const base = await this.backend.requestPermission(permission);
+            return { ...base, settings: this.settings() };
+        });
+    }
+    async status() {
+        const base = await this.backend.status();
+        return { ...base, settings: this.settings() };
+    }
+    setSettings(patch) {
+        const next = this.settingsRepository.set({ ...this.settings(), ...patch });
+        if (!next.enabled) {
+            this.currentSnapshotId = null;
+            this.latestSnapshotValue = null;
+            void this.backend.dispose().catch(() => undefined);
+        }
+        this.events.emit({ type: "computer-settings", settings: next });
+        return next;
+    }
+    snapshot(scope, request) {
+        return this.serial(async () => {
+            this.assertAllowed(scope);
+            this.currentSnapshotId = null;
+            const value = await this.backend.snapshot(request, this.revision);
+            this.currentSnapshotId = value.snapshotId || `snap_${randomUUID()}`;
+            value.snapshotId = this.currentSnapshotId;
+            value.revision = this.revision;
+            this.latestSnapshotValue = value;
+            return value;
+        });
+    }
+    action(scope, request) {
+        return this.serial(async () => {
+            this.assertAllowed(scope);
+            this.assertActionBudget(request);
+            const referencesElements = request.actions.some((action) => "elementId" in action && action.elementId !== undefined);
+            if (referencesElements && !request.snapshotId)
+                throw new ChatRoomError("CONFLICT", "Actions using elementId require the latest snapshotId");
+            if (request.snapshotId && request.snapshotId !== this.currentSnapshotId)
+                throw new ChatRoomError("CONFLICT", "Computer snapshot is stale; take a new snapshot before acting");
+            this.currentSnapshotId = null;
+            const result = await this.backend.action(request, ++this.revision);
+            this.currentSnapshotId = result.snapshot?.snapshotId ?? null;
+            if (result.snapshot)
+                this.latestSnapshotValue = result.snapshot;
+            return result;
+        });
+    }
+    async shutdown() {
+        this.currentSnapshotId = null;
+        this.latestSnapshotValue = null;
+        await this.backend.dispose();
+    }
+    assertActionBudget(request) {
+        const durationMs = request.actions.reduce((total, action) => {
+            if (action.type === "wait")
+                return total + action.ms;
+            if (action.type === "drag")
+                return total + (action.durationMs ?? 0);
+            return total;
+        }, 0);
+        if (durationMs > 30_000)
+            throw new ChatRoomError("INVALID_INPUT", "Computer action batch exceeds the 30 second execution budget");
+    }
+    assertAllowed(scope) {
+        const settings = this.settings();
+        if (!settings.enabled)
+            throw new ChatRoomError("FORBIDDEN", "Computer Use is disabled");
+        this.assertScopeAllowed(scope, settings);
+    }
+    assertScopeAllowed(scope, settings) {
+        if (scope === "remote" && !settings.remoteAccess)
+            throw new ChatRoomError("FORBIDDEN", "Remote Computer Use is disabled", {
+                reason: "remote_computer_disabled",
+            });
+    }
+    serial(action) {
+        const next = this.queue.then(action, action);
+        this.queue = next.then(() => undefined, () => undefined);
+        return next;
+    }
+}

package/dist/plugins/computer/computer-settings-repository.d.ts

@@ -0,0 +1,8 @@
+import type { AppDatabase } from "../../infrastructure/database/app-database.js";
+import type { ComputerSettings } from "./types.js";
+export declare class ComputerSettingsRepository {
+    private readonly database;
+    constructor(database: AppDatabase);
+    get(): ComputerSettings;
+    set(value: Omit<ComputerSettings, "updatedAt"> | ComputerSettings): ComputerSettings;
+}

package/dist/plugins/computer/computer-settings-repository.js

@@ -0,0 +1,41 @@
+export class ComputerSettingsRepository {
+    database;
+    constructor(database) {
+        this.database = database;
+    }
+    get() {
+        const row = this.database.raw
+            .prepare("SELECT enabled, remote_access, updated_at FROM computer_settings WHERE id = 1")
+            .get();
+        if (row)
+            return fromRow(row);
+        const value = {
+            enabled: false,
+            remoteAccess: true,
+            updatedAt: new Date().toISOString(),
+        };
+        this.set(value);
+        return value;
+    }
+    set(value) {
+        const updatedAt = new Date().toISOString();
+        this.database.raw
+            .prepare(`
+      INSERT INTO computer_settings(id, enabled, remote_access, updated_at)
+      VALUES(1, ?, ?, ?)
+      ON CONFLICT(id) DO UPDATE SET
+        enabled = excluded.enabled,
+        remote_access = excluded.remote_access,
+        updated_at = excluded.updated_at
+    `)
+            .run(value.enabled ? 1 : 0, value.remoteAccess ? 1 : 0, updatedAt);
+        return { ...value, updatedAt };
+    }
+}
+function fromRow(row) {
+    return {
+        enabled: Boolean(row.enabled),
+        remoteAccess: Boolean(row.remote_access),
+        updatedAt: String(row.updated_at),
+    };
+}

package/dist/plugins/computer/mcp.d.ts

@@ -0,0 +1,3 @@
+import type { PluginMcpRegistrar } from "../../mcp/server/plugin-mcp-registrar.js";
+import type { ComputerService } from "./computer-service.js";
+export declare function registerComputerTools(mcp: PluginMcpRegistrar, service: ComputerService): void;

package/dist/plugins/computer/mcp.js

@@ -0,0 +1,126 @@
+import { z } from "zod";
+import { closedRead, openWorldMutation, } from "../../mcp/server/tool-support.js";
+import { currentMcpAccessScope } from "../../mcp/server/request-context.js";
+import { auditActionOutput, auditComputerActions, auditSnapshotOutput, } from "./audit.js";
+import { computerActionMetadataSchema, computerActionSchema, computerSnapshotMetadataSchema, computerSnapshotTargetSchema, } from "./computer-schemas.js";
+export function registerComputerTools(mcp, service) {
+    mcp.registerTool("computer_snapshot", {
+        title: "Observe computer",
+        description: "Observe the current desktop using a screenshot and compact accessibility elements. Element IDs are valid only for the returned snapshotId.",
+        inputSchema: z.object({
+            target: computerSnapshotTargetSchema.optional(),
+            includeScreenshot: z.boolean().default(true),
+            includeElements: z.boolean().default(true),
+        }),
+        outputSchema: computerSnapshotMetadataSchema,
+        annotations: closedRead,
+        action: "snapshot",
+        audit: {
+            output: (value) => auditSnapshotOutput(value),
+        },
+        present: presentSnapshot,
+    }, (input) => service.snapshot(currentMcpAccessScope(), {
+        includeScreenshot: input.includeScreenshot,
+        includeElements: input.includeElements,
+        ...(input.target === undefined
+            ? {}
+            : { target: normalizeSnapshotTarget(input.target) }),
+    }));
+    mcp.registerTool("computer_action", {
+        title: "Control computer",
+        description: "Execute a bounded batch of semantic or coordinate desktop actions. Prefer elementId from the latest snapshot; elementId actions require that snapshotId.",
+        inputSchema: z.object({
+            snapshotId: z.string().optional(),
+            actions: z.array(computerActionSchema).min(1).max(50),
+            observeAfter: z.boolean().default(true),
+        }),
+        outputSchema: computerActionMetadataSchema,
+        annotations: openWorldMutation,
+        action: "action",
+        audit: {
+            input: (value) => auditComputerActions(value),
+            output: (value) => auditActionOutput(value),
+        },
+        present: presentAction,
+    }, (input) => service.action(currentMcpAccessScope(), {
+        actions: input.actions,
+        observeAfter: input.observeAfter,
+        ...(input.snapshotId === undefined
+            ? {}
+            : { snapshotId: input.snapshotId }),
+    }));
+}
+function presentSnapshot(value) {
+    const snapshot = value;
+    const { screenshot, ...metadata } = snapshot;
+    return {
+        content: [
+            { type: "text", text: JSON.stringify(metadata, null, 2) },
+            ...(screenshot
+                ? [
+                    {
+                        type: "image",
+                        data: screenshot.data,
+                        mimeType: screenshot.mimeType,
+                    },
+                ]
+                : []),
+        ],
+        structuredContent: metadata,
+    };
+}
+function presentAction(value) {
+    const result = value;
+    const screenshot = result.snapshot?.screenshot;
+    const snapshot = result.snapshot
+        ? (({ screenshot: _screenshot, ...metadata }) => metadata)(result.snapshot)
+        : undefined;
+    const safe = {
+        success: result.success,
+        revision: result.revision,
+        executionMode: result.executionMode,
+        focusChanged: result.focusChanged,
+        ...(snapshot === undefined ? {} : { snapshot }),
+    };
+    return {
+        content: [
+            { type: "text", text: JSON.stringify(safe, null, 2) },
+            ...(screenshot
+                ? [
+                    {
+                        type: "image",
+                        data: screenshot.data,
+                        mimeType: screenshot.mimeType,
+                    },
+                ]
+                : []),
+        ],
+        structuredContent: safe,
+    };
+}
+function normalizeSnapshotTarget(input) {
+    switch (input.type) {
+        case "desktop":
+            return { type: "desktop" };
+        case "display":
+            return input.displayId === undefined
+                ? { type: "display" }
+                : { type: "display", displayId: input.displayId };
+        case "app":
+            return { type: "app", app: input.app };
+        case "window":
+            return { type: "window", elementId: input.elementId };
+        case "region": {
+            const region = {
+                type: "region",
+                x: input.x,
+                y: input.y,
+                width: input.width,
+                height: input.height,
+            };
+            return input.displayId === undefined
+                ? region
+                : { ...region, displayId: input.displayId };
+        }
+    }
+}

package/dist/plugins/computer/plugin.d.ts

@@ -0,0 +1,4 @@
+import type { InternalPlugin } from "../types.js";
+import { ComputerService } from "./computer-service.js";
+export declare const ComputerServiceToken: import("../types.js").ServiceToken<ComputerService>;
+export declare function createComputerPlugin(): InternalPlugin;

package/dist/plugins/computer/plugin.js

@@ -0,0 +1,29 @@
+import { createServiceToken } from "../types.js";
+import { NativeComputerBackend } from "./computer-native-backend.js";
+import { ComputerService } from "./computer-service.js";
+import { ComputerSettingsRepository } from "./computer-settings-repository.js";
+import { registerComputerTools } from "./mcp.js";
+import { currentMcpAccessScope } from "../../mcp/server/request-context.js";
+export const ComputerServiceToken = createServiceToken("computer");
+export function createComputerPlugin() {
+    let service = null;
+    return {
+        id: "computer",
+        activate(context) {
+            service = new ComputerService(new NativeComputerBackend(), new ComputerSettingsRepository(context.database), context.events);
+            context.services.provide(ComputerServiceToken, service);
+        },
+        registerMcp(mcp) {
+            if (!service)
+                throw new Error("Computer plugin is not active");
+            if (currentMcpAccessScope() === "remote" &&
+                !service.settings().remoteAccess)
+                return;
+            registerComputerTools(mcp, service);
+        },
+        async deactivate() {
+            await service?.shutdown();
+            service = null;
+        },
+    };
+}

package/dist/plugins/computer/types.d.ts

@@ -0,0 +1,174 @@
+export type ComputerAccessScope = "local" | "remote";
+export type ComputerPlatform = "macos" | "windows" | "unsupported";
+export type ComputerHelperState = "running" | "stopped" | "unavailable";
+export type ComputerPermissionState = "granted" | "denied" | "unknown" | "not-required";
+export type ComputerPermission = "accessibility" | "screenRecording";
+export interface ComputerSettings {
+    enabled: boolean;
+    remoteAccess: boolean;
+    updatedAt: string;
+}
+export interface ComputerPermissions {
+    accessibility: ComputerPermissionState;
+    screenRecording: ComputerPermissionState;
+}
+export interface ComputerDisplay {
+    id: string;
+    name: string;
+    width: number;
+    height: number;
+    scale: number;
+    primary: boolean;
+}
+export interface ComputerStatus {
+    platform: ComputerPlatform;
+    helper: ComputerHelperState;
+    permissions: ComputerPermissions;
+    displays: ComputerDisplay[];
+    settings: ComputerSettings;
+}
+export interface ComputerElement {
+    id: number;
+    role: string;
+    name: string | null;
+    value: string | null;
+    enabled: boolean;
+    focused: boolean;
+    selected: boolean;
+    sensitive: boolean;
+    bounds: [number, number, number, number] | null;
+    actions: string[];
+}
+export type ComputerSnapshotTarget = {
+    type: "desktop";
+} | {
+    type: "display";
+    displayId?: string;
+} | {
+    type: "app";
+    app: string;
+} | {
+    type: "window";
+    elementId: number;
+} | {
+    type: "region";
+    displayId?: string;
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+};
+export interface ComputerSnapshotRequest {
+    target?: ComputerSnapshotTarget;
+    includeScreenshot: boolean;
+    includeElements: boolean;
+}
+export interface ComputerSnapshot {
+    snapshotId: string;
+    revision: number;
+    display: ComputerDisplay | null;
+    activeApp: string | null;
+    activeWindow: string | null;
+    cursor: {
+        x: number;
+        y: number;
+    } | null;
+    elements: ComputerElement[];
+    screenshot?: {
+        mimeType: "image/jpeg" | "image/png";
+        data: string;
+    };
+}
+export type ComputerAction = {
+    type: "move";
+    x: number;
+    y: number;
+} | {
+    type: "click";
+    x?: number;
+    y?: number;
+    elementId?: number;
+} | {
+    type: "double_click";
+    x?: number;
+    y?: number;
+    elementId?: number;
+} | {
+    type: "right_click";
+    x?: number;
+    y?: number;
+    elementId?: number;
+} | {
+    type: "drag";
+    from: {
+        x: number;
+        y: number;
+    };
+    to: {
+        x: number;
+        y: number;
+    };
+    durationMs?: number;
+} | {
+    type: "scroll";
+    deltaX?: number;
+    deltaY: number;
+    elementId?: number;
+} | {
+    type: "keypress";
+    keys: string[];
+} | {
+    type: "type_text";
+    text: string;
+    elementId?: number;
+} | {
+    type: "invoke";
+    elementId: number;
+} | {
+    type: "set_value";
+    elementId: number;
+    value: string;
+} | {
+    type: "select_text";
+    elementId: number;
+    start: number;
+    length: number;
+} | {
+    type: "activate_app";
+    app: string;
+} | {
+    type: "activate_window";
+    elementId: number;
+} | {
+    type: "move_window";
+    elementId: number;
+    x: number;
+    y: number;
+} | {
+    type: "resize_window";
+    elementId: number;
+    width: number;
+    height: number;
+} | {
+    type: "wait";
+    ms: number;
+};
+export interface ComputerActionRequest {
+    snapshotId?: string;
+    actions: ComputerAction[];
+    observeAfter: boolean;
+}
+export interface ComputerActionResult {
+    success: true;
+    revision: number;
+    snapshot?: ComputerSnapshot;
+    executionMode: "semantic" | "background" | "foreground" | "mixed";
+    focusChanged: boolean;
+}
+export interface ComputerBackend {
+    status(): Promise<Omit<ComputerStatus, "settings">>;
+    requestPermission(permission: ComputerPermission): Promise<Omit<ComputerStatus, "settings">>;
+    snapshot(request: ComputerSnapshotRequest, revision: number): Promise<ComputerSnapshot>;
+    action(request: ComputerActionRequest, revision: number): Promise<ComputerActionResult>;
+    dispose(): Promise<void>;
+}

package/dist/plugins/computer/types.js

@@ -0,0 +1 @@
+export {};