agent-vision-mcp 0.1.0

package/dist/mcp/stdio-server.js

@@ -0,0 +1,272 @@
+import { isAppError } from "../errors/app-error.js";
+const MCP_PROTOCOL_VERSION = "2025-03-26";
+const SERVER_NAME = "agent-vision-mcp";
+const SERVER_VERSION = "0.1.0";
+const JSON_RPC_VERSION = "2.0";
+const JSON_PARSE_ERROR = -32700;
+const JSON_INVALID_REQUEST = -32600;
+const JSON_METHOD_NOT_FOUND = -32601;
+const JSON_INVALID_PARAMS = -32602;
+const JSON_INTERNAL_ERROR = -32603;
+const toRecord = (value) => {
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+        return {};
+    }
+    return value;
+};
+const isImagePayload = (value) => {
+    if (!value || typeof value !== "object") {
+        return false;
+    }
+    const candidate = value;
+    return typeof candidate.mimeType === "string" && typeof candidate.bytesBase64 === "string";
+};
+const collectImagePayloads = (value, seen = new Set()) => {
+    if (!value || typeof value !== "object") {
+        return [];
+    }
+    if (seen.has(value)) {
+        return [];
+    }
+    seen.add(value);
+    if (isImagePayload(value)) {
+        return [{ mimeType: value.mimeType, bytesBase64: value.bytesBase64 }];
+    }
+    if (Array.isArray(value)) {
+        return value.flatMap((item) => collectImagePayloads(item, seen));
+    }
+    return Object.values(value).flatMap((item) => collectImagePayloads(item, seen));
+};
+const redactImageBytes = (value, seen = new Set()) => {
+    if (!value || typeof value !== "object") {
+        return value;
+    }
+    if (seen.has(value)) {
+        return "[circular]";
+    }
+    seen.add(value);
+    if (Array.isArray(value)) {
+        return value.map((item) => redactImageBytes(item, seen));
+    }
+    const output = {};
+    for (const [key, item] of Object.entries(value)) {
+        if (key === "bytesBase64" && typeof item === "string") {
+            output[key] = `[base64 omitted: ${item.length} chars]`;
+            continue;
+        }
+        output[key] = redactImageBytes(item, seen);
+    }
+    return output;
+};
+const toToolResult = (result) => {
+    const content = [
+        {
+            type: "text",
+            text: JSON.stringify(redactImageBytes(result), null, 2)
+        }
+    ];
+    for (const image of collectImagePayloads(result)) {
+        content.push({
+            type: "image",
+            mimeType: image.mimeType,
+            data: image.bytesBase64
+        });
+    }
+    return { content };
+};
+const toToolErrorResult = (error) => {
+    const message = isAppError(error)
+        ? `${error.code}: ${error.message}`
+        : error instanceof Error
+            ? error.message
+            : String(error);
+    const details = isAppError(error) ? error.details : undefined;
+    return {
+        content: [
+            {
+                type: "text",
+                text: JSON.stringify({
+                    error: message,
+                    ...(details ? { details } : {})
+                }, null, 2)
+            }
+        ],
+        isError: true
+    };
+};
+const toJsonRpcError = (id, code, message, data) => ({
+    jsonrpc: JSON_RPC_VERSION,
+    id,
+    error: {
+        code,
+        message,
+        ...(data !== undefined ? { data } : {})
+    }
+});
+const encodeMessage = (message) => {
+    const body = Buffer.from(JSON.stringify(message), "utf8");
+    const header = Buffer.from(`Content-Length: ${body.byteLength}\r\n\r\n`, "utf8");
+    return Buffer.concat([header, body]);
+};
+export class McpStdioServer {
+    app;
+    logger;
+    input;
+    output;
+    buffer = Buffer.alloc(0);
+    initialized = false;
+    constructor(app, logger, input = process.stdin, output = process.stdout) {
+        this.app = app;
+        this.logger = logger;
+        this.input = input;
+        this.output = output;
+    }
+    start() {
+        this.input.on("data", (chunk) => {
+            this.buffer = Buffer.concat([
+                this.buffer,
+                typeof chunk === "string" ? Buffer.from(chunk, "utf8") : chunk
+            ]);
+            this.processBuffer();
+        });
+        this.input.on("error", (error) => {
+            this.logger.error("MCP stdio input error", {
+                errorMessage: error instanceof Error ? error.message : String(error)
+            });
+        });
+        this.output.on("error", (error) => {
+            this.logger.error("MCP stdio output error", {
+                errorMessage: error instanceof Error ? error.message : String(error)
+            });
+        });
+        this.logger.info("MCP stdio server listening", {
+            protocolVersion: MCP_PROTOCOL_VERSION,
+            serverName: SERVER_NAME,
+            serverVersion: SERVER_VERSION
+        });
+    }
+    processBuffer() {
+        while (true) {
+            const headerEnd = this.buffer.indexOf("\r\n\r\n");
+            if (headerEnd === -1) {
+                return;
+            }
+            const headerText = this.buffer.subarray(0, headerEnd).toString("utf8");
+            const match = headerText.match(/Content-Length:\s*(\d+)/i);
+            if (!match) {
+                this.buffer = Buffer.alloc(0);
+                this.writeMessage(toJsonRpcError(null, JSON_INVALID_REQUEST, "Missing Content-Length header"));
+                return;
+            }
+            const contentLength = Number.parseInt(match[1], 10);
+            const bodyStart = headerEnd + 4;
+            const bodyEnd = bodyStart + contentLength;
+            if (this.buffer.byteLength < bodyEnd) {
+                return;
+            }
+            const bodyBuffer = this.buffer.subarray(bodyStart, bodyEnd);
+            this.buffer = this.buffer.subarray(bodyEnd);
+            let parsed;
+            try {
+                parsed = JSON.parse(bodyBuffer.toString("utf8"));
+            }
+            catch {
+                this.writeMessage(toJsonRpcError(null, JSON_PARSE_ERROR, "Invalid JSON payload"));
+                continue;
+            }
+            void this.handleMessage(parsed);
+        }
+    }
+    async handleMessage(message) {
+        if (message.jsonrpc !== JSON_RPC_VERSION || typeof message.method !== "string") {
+            if (message.id !== undefined) {
+                this.writeMessage(toJsonRpcError(message.id ?? null, JSON_INVALID_REQUEST, "Invalid JSON-RPC request"));
+            }
+            return;
+        }
+        const id = message.id ?? null;
+        try {
+            switch (message.method) {
+                case "initialize": {
+                    this.initialized = true;
+                    this.writeMessage({
+                        jsonrpc: JSON_RPC_VERSION,
+                        id,
+                        result: {
+                            protocolVersion: MCP_PROTOCOL_VERSION,
+                            capabilities: {
+                                tools: {
+                                    listChanged: false
+                                }
+                            },
+                            serverInfo: {
+                                name: SERVER_NAME,
+                                version: SERVER_VERSION
+                            }
+                        }
+                    });
+                    return;
+                }
+                case "notifications/initialized": {
+                    return;
+                }
+                case "ping": {
+                    this.writeMessage({
+                        jsonrpc: JSON_RPC_VERSION,
+                        id,
+                        result: {}
+                    });
+                    return;
+                }
+                case "tools/list": {
+                    this.writeMessage({
+                        jsonrpc: JSON_RPC_VERSION,
+                        id,
+                        result: {
+                            tools: this.app.listTools().map((tool) => ({
+                                name: tool.name,
+                                description: tool.description,
+                                inputSchema: tool.inputSchema ?? { type: "object", properties: {}, additionalProperties: false },
+                                ...(tool.annotations ? { annotations: tool.annotations } : {})
+                            }))
+                        }
+                    });
+                    return;
+                }
+                case "tools/call": {
+                    const params = toRecord(message.params);
+                    const toolName = params.name;
+                    if (typeof toolName !== "string") {
+                        this.writeMessage(toJsonRpcError(id, JSON_INVALID_PARAMS, "tools/call requires a string name"));
+                        return;
+                    }
+                    try {
+                        const result = await this.app.callTool(toolName, toRecord(params.arguments));
+                        this.writeMessage({
+                            jsonrpc: JSON_RPC_VERSION,
+                            id,
+                            result: toToolResult(result)
+                        });
+                    }
+                    catch (error) {
+                        this.writeMessage({
+                            jsonrpc: JSON_RPC_VERSION,
+                            id,
+                            result: toToolErrorResult(error)
+                        });
+                    }
+                    return;
+                }
+                default: {
+                    this.writeMessage(toJsonRpcError(id, JSON_METHOD_NOT_FOUND, `Method not found: ${message.method}`));
+                }
+            }
+        }
+        catch (error) {
+            this.writeMessage(toJsonRpcError(id, JSON_INTERNAL_ERROR, "Internal MCP server error", error instanceof Error ? { message: error.message } : { message: String(error) }));
+        }
+    }
+    writeMessage(message) {
+        this.output.write(encodeMessage(message));
+    }
+}

package/dist/mcp/tool-registry.d.ts

@@ -0,0 +1,21 @@
+import type { Logger } from "../logging/logger.js";
+export type JsonSchema = Record<string, unknown>;
+export type ToolHandler = (args: Record<string, unknown>) => Promise<unknown> | unknown;
+export type ToolAnnotations = {
+    readOnlyHint?: boolean;
+};
+export type ToolDefinition = {
+    name: string;
+    description: string;
+    inputSchema?: JsonSchema;
+    annotations?: ToolAnnotations;
+    handler: ToolHandler;
+};
+export declare class ToolRegistry {
+    private readonly logger;
+    private readonly tools;
+    constructor(logger: Logger);
+    register(definition: ToolDefinition): void;
+    list(): Omit<ToolDefinition, "handler">[];
+    call(name: string, args?: Record<string, unknown>): Promise<unknown>;
+}

package/dist/mcp/tool-registry.js

@@ -0,0 +1,33 @@
+import { AppError, isAppError } from "../errors/app-error.js";
+export class ToolRegistry {
+    logger;
+    tools = new Map();
+    constructor(logger) {
+        this.logger = logger;
+    }
+    register(definition) {
+        this.tools.set(definition.name, definition);
+    }
+    list() {
+        return Array.from(this.tools.values()).map(({ handler: _handler, ...tool }) => tool);
+    }
+    async call(name, args = {}) {
+        const definition = this.tools.get(name);
+        if (!definition) {
+            throw new AppError("NOT_FOUND", "Tool not found", { toolName: name });
+        }
+        try {
+            this.logger.debug("Calling tool", { toolName: name });
+            return await definition.handler(args);
+        }
+        catch (error) {
+            if (isAppError(error)) {
+                throw error;
+            }
+            throw new AppError("INTERNAL_ERROR", "Unexpected tool failure", {
+                toolName: name,
+                cause: error instanceof Error ? error.message : String(error)
+            });
+        }
+    }
+}

package/dist/mcp-stdio.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/mcp-stdio.js

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+import { ConsoleLogger } from "./logging/logger.js";
+import { McpStdioServer } from "./mcp/stdio-server.js";
+import { VisualContextServer } from "./server.js";
+const app = new VisualContextServer();
+const logger = new ConsoleLogger("agent-vision-mcp");
+app.start();
+new McpStdioServer(app, logger).start();

package/dist/overlay/local-overlay-agent.d.ts

@@ -0,0 +1,46 @@
+import type { CapturePipeline } from "../capture/capture-pipeline.js";
+import type { Logger } from "../logging/logger.js";
+import { CaptureSessionService } from "../session/capture-session-service.js";
+import type { CaptureSession } from "../types/session.js";
+import type { CreateAnnotationInput, OverlayMoveInput, OverlayRegionInput, OverlayResizeInput, OverlaySession, OverlayTool, UpdateAnnotationInput } from "./types.js";
+export declare class LocalOverlayAgent {
+    private readonly captureSessions;
+    private readonly capturePipeline;
+    private readonly logger;
+    private readonly overlaySessions;
+    constructor(captureSessions: CaptureSessionService, capturePipeline: CapturePipeline, logger: Logger);
+    launch(sessionId: string): OverlaySession;
+    get(sessionId: string): OverlaySession;
+    list(): OverlaySession[];
+    setActiveTool(sessionId: string, tool: OverlayTool): OverlaySession;
+    selectRegion(sessionId: string, input: OverlayRegionInput): OverlaySession;
+    moveSelection(sessionId: string, input: OverlayMoveInput): OverlaySession;
+    resizeSelection(sessionId: string, input: OverlayResizeInput): OverlaySession;
+    addAnnotation(sessionId: string, input: CreateAnnotationInput): OverlaySession;
+    updateAnnotation(sessionId: string, input: UpdateAnnotationInput): OverlaySession;
+    removeAnnotation(sessionId: string, annotationId: string): OverlaySession;
+    clearAnnotations(sessionId: string): OverlaySession;
+    send(sessionId: string): Promise<CaptureSession>;
+    cancel(sessionId: string): CaptureSession;
+    fail(sessionId: string, errorMessage: string): CaptureSession;
+    reapTerminalSessions(maxAgeMs: number): {
+        removedSessionIds: string[];
+    };
+    private requireSelectedOverlaySession;
+    private requireMutableOverlaySession;
+    private requireOverlaySession;
+    private syncWithCaptureSession;
+    private persist;
+    private present;
+    private buildSelectionSummary;
+    private buildAnnotationSummary;
+    private buildPreviewState;
+    private buildHudState;
+    private assertValidBounds;
+    private assertAnnotation;
+    private assertBoxAnnotation;
+    private assertRedactAnnotation;
+    private assertArrowAnnotation;
+    private assertTextAnnotation;
+    private isTerminal;
+}