agent-vision-mcp 0.1.0

package/dist/server.js

@@ -0,0 +1,158 @@
+import { BrowserCdpDiscoveryService } from "./browser/cdp/browser-cdp-discovery-service.js";
+import { BrowserLiveTabService } from "./browser/cdp/browser-live-tab-service.js";
+import { BrowserSeeService } from "./browser/cdp/browser-see-service.js";
+import { BrowserTabContextService } from "./browser/cdp/browser-tab-context-service.js";
+import { BrowserTabResolutionService } from "./browser/cdp/browser-tab-resolution-service.js";
+import { BrowserTabScreenshotService } from "./browser/cdp/browser-tab-screenshot-service.js";
+import { ChromeCdpClient } from "./browser/cdp/chrome-cdp-client.js";
+import { LiveBrowserTabRegistry } from "./browser/cdp/live-browser-tab-registry.js";
+import { AppError } from "./errors/app-error.js";
+import { ConsoleLogger } from "./logging/logger.js";
+import { ToolRegistry } from "./mcp/tool-registry.js";
+const STRING_SCHEMA = (description) => ({
+    type: "string",
+    description
+});
+const NUMBER_SCHEMA = (description) => ({
+    type: "number",
+    description
+});
+const OPTIONAL_QUERY_SCHEMA = {
+    type: "object",
+    properties: {
+        query: STRING_SCHEMA("Optional tab title or URL fragment to match against live browser tabs.")
+    },
+    additionalProperties: false
+};
+const OPTIONAL_ENDPOINT_SCHEMA = {
+    type: "object",
+    properties: {
+        endpoint: STRING_SCHEMA("Optional Chrome DevTools Protocol base URL, for example http://127.0.0.1:9222.")
+    },
+    additionalProperties: false
+};
+const OPTIONAL_PRUNE_SCHEMA = {
+    type: "object",
+    properties: {
+        maxAgeMs: NUMBER_SCHEMA("Optional stale-tab age threshold in milliseconds.")
+    },
+    additionalProperties: false
+};
+const EMPTY_OBJECT_SCHEMA = {
+    type: "object",
+    properties: {},
+    additionalProperties: false
+};
+const READ_ONLY_ANNOTATIONS = {
+    readOnlyHint: true
+};
+const readOptionalString = (value, field) => {
+    if (value === undefined) {
+        return undefined;
+    }
+    if (typeof value !== "string") {
+        throw new AppError("INVALID_ARGUMENT", `${field} must be a string`);
+    }
+    return value;
+};
+const readOptionalEndpoint = (args) => readOptionalString(args.endpoint, "endpoint");
+const readOptionalNumber = (value, field) => {
+    if (value === undefined) {
+        return undefined;
+    }
+    if (typeof value !== "number" || Number.isNaN(value) || value < 0) {
+        throw new AppError("INVALID_ARGUMENT", `${field} must be a non-negative number`);
+    }
+    return value;
+};
+export class VisualContextServer {
+    logger = new ConsoleLogger("visual-context-server");
+    cdpClient = new ChromeCdpClient(this.logger);
+    cdpDiscovery = new BrowserCdpDiscoveryService(this.cdpClient, this.logger);
+    liveTabs = new LiveBrowserTabRegistry(this.logger);
+    browserLiveTabs = new BrowserLiveTabService(this.cdpDiscovery, this.liveTabs, this.logger);
+    tabResolution = new BrowserTabResolutionService(this.browserLiveTabs, this.logger);
+    tabScreenshots = new BrowserTabScreenshotService(this.tabResolution, this.logger);
+    tabContext = new BrowserTabContextService(this.tabResolution, this.logger);
+    browserSee = new BrowserSeeService(this.browserLiveTabs, this.tabScreenshots, this.tabContext, this.logger);
+    tools = new ToolRegistry(this.logger);
+    constructor() {
+        this.registerTools();
+    }
+    listTools() {
+        return this.tools.list();
+    }
+    async callTool(name, args = {}) {
+        return this.tools.call(name, args);
+    }
+    start() {
+        this.logger.info("CDP-first visual context server ready", {
+            tools: this.listTools().map((tool) => tool.name)
+        });
+    }
+    registerTools() {
+        this.tools.register({
+            name: "getBrowserCdpStatus",
+            description: "Use this when you need to verify that the MCP can reach a Chrome DevTools Protocol endpoint.",
+            inputSchema: OPTIONAL_ENDPOINT_SCHEMA,
+            annotations: READ_ONLY_ANNOTATIONS,
+            handler: (args) => this.cdpDiscovery.getConnectionStatus(readOptionalEndpoint(args))
+        });
+        this.tools.register({
+            name: "discoverBrowserTabsViaCdp",
+            description: "Use this when you need the raw list of browser tabs currently exposed by Chrome DevTools Protocol.",
+            inputSchema: OPTIONAL_ENDPOINT_SCHEMA,
+            annotations: READ_ONLY_ANNOTATIONS,
+            handler: (args) => this.cdpDiscovery.discoverTabs(readOptionalEndpoint(args))
+        });
+        this.tools.register({
+            name: "refreshLiveBrowserTabs",
+            description: "Use this when you want to refresh the normalized live browser-tab cache from Chrome DevTools Protocol.",
+            inputSchema: OPTIONAL_ENDPOINT_SCHEMA,
+            annotations: READ_ONLY_ANNOTATIONS,
+            handler: (args) => this.browserLiveTabs.refresh(readOptionalEndpoint(args))
+        });
+        this.tools.register({
+            name: "listLiveBrowserTabs",
+            description: "Use this when you want the current cached live browser-tab model without re-querying Chrome.",
+            inputSchema: EMPTY_OBJECT_SCHEMA,
+            annotations: READ_ONLY_ANNOTATIONS,
+            handler: () => this.browserLiveTabs.list()
+        });
+        this.tools.register({
+            name: "pruneStaleLiveBrowserTabs",
+            description: "Use this when you want to remove stale cached browser tabs that have not been refreshed recently.",
+            inputSchema: OPTIONAL_PRUNE_SCHEMA,
+            annotations: READ_ONLY_ANNOTATIONS,
+            handler: (args) => this.browserLiveTabs.pruneStale(readOptionalNumber(args.maxAgeMs, "maxAgeMs"))
+        });
+        this.tools.register({
+            name: "resolveLiveBrowserTab",
+            description: "Use this when you want to resolve the active or best matching browser tab for a /see-style query.",
+            inputSchema: OPTIONAL_QUERY_SCHEMA,
+            annotations: READ_ONLY_ANNOTATIONS,
+            handler: (args) => this.tabResolution.resolve(readOptionalString(args.query, "query"))
+        });
+        this.tools.register({
+            name: "captureResolvedBrowserTabScreenshot",
+            description: "Use this when you need a real PNG screenshot from the resolved live browser tab through CDP.",
+            inputSchema: OPTIONAL_QUERY_SCHEMA,
+            annotations: READ_ONLY_ANNOTATIONS,
+            handler: (args) => this.tabScreenshots.captureResolved(readOptionalString(args.query, "query"))
+        });
+        this.tools.register({
+            name: "getResolvedBrowserTabContext",
+            description: "Use this when you need structured page metadata and visible text from the resolved live browser tab.",
+            inputSchema: OPTIONAL_QUERY_SCHEMA,
+            annotations: READ_ONLY_ANNOTATIONS,
+            handler: (args) => this.tabContext.getResolvedContext(readOptionalString(args.query, "query"))
+        });
+        this.tools.register({
+            name: "seeBrowserTabViaCdp",
+            description: "Use this for the high-level browser-first /see flow: resolve a live tab, capture it, and return structured page context.",
+            inputSchema: OPTIONAL_QUERY_SCHEMA,
+            annotations: READ_ONLY_ANNOTATIONS,
+            handler: (args) => this.browserSee.see(readOptionalString(args.query, "query"))
+        });
+    }
+}

package/dist/session/capture-session-service.d.ts

@@ -0,0 +1,21 @@
+import type { Logger } from "../logging/logger.js";
+import type { AwaitCaptureSessionInput, AwaitCaptureSessionResult, CaptureSession, CompleteCaptureSessionInput, StartCaptureSessionInput } from "../types/session.js";
+import { SessionManager } from "./session-manager.js";
+import { SessionWaiter } from "./session-waiter.js";
+export declare class CaptureSessionService {
+    private readonly sessionManager;
+    private readonly sessionWaiter;
+    private readonly logger;
+    constructor(sessionManager: SessionManager, sessionWaiter: SessionWaiter, logger: Logger);
+    startSession(input: StartCaptureSessionInput): CaptureSession;
+    awaitSession(input: AwaitCaptureSessionInput): Promise<AwaitCaptureSessionResult>;
+    getSession(sessionId: string): CaptureSession;
+    listSessions(): CaptureSession[];
+    completeSession(input: CompleteCaptureSessionInput): CaptureSession;
+    cancelSession(sessionId: string): CaptureSession;
+    failSession(sessionId: string, errorMessage: string): CaptureSession;
+    reapTerminalSessions(maxAgeMs: number): {
+        removedSessionIds: string[];
+    };
+    logStateSummary(): void;
+}

package/dist/session/capture-session-service.js

@@ -0,0 +1,50 @@
+export class CaptureSessionService {
+    sessionManager;
+    sessionWaiter;
+    logger;
+    constructor(sessionManager, sessionWaiter, logger) {
+        this.sessionManager = sessionManager;
+        this.sessionWaiter = sessionWaiter;
+        this.logger = logger;
+    }
+    startSession(input) {
+        const session = this.sessionManager.startSession(input);
+        return this.sessionManager.activateSession(session.id);
+    }
+    awaitSession(input) {
+        return this.sessionWaiter.awaitSession(input);
+    }
+    getSession(sessionId) {
+        return this.sessionManager.getSession(sessionId);
+    }
+    listSessions() {
+        return this.sessionManager.listSessions();
+    }
+    completeSession(input) {
+        const completed = this.sessionManager.completeSession(input);
+        this.sessionWaiter.notify(completed);
+        return completed;
+    }
+    cancelSession(sessionId) {
+        const cancelled = this.sessionManager.cancelSession(sessionId);
+        this.sessionWaiter.notify(cancelled);
+        return cancelled;
+    }
+    failSession(sessionId, errorMessage) {
+        const failed = this.sessionManager.failSession(sessionId, errorMessage);
+        this.sessionWaiter.notify(failed);
+        return failed;
+    }
+    reapTerminalSessions(maxAgeMs) {
+        const result = this.sessionManager.reapTerminalSessions(maxAgeMs);
+        for (const sessionId of result.removedSessionIds) {
+            this.sessionWaiter.clearSession(sessionId);
+        }
+        return result;
+    }
+    logStateSummary() {
+        this.logger.debug("Capture session service state", {
+            sessions: this.listSessions().length
+        });
+    }
+}

package/dist/session/session-manager.d.ts

@@ -0,0 +1,29 @@
+import type { Logger } from "../logging/logger.js";
+import type { CaptureBundle } from "../types/capture.js";
+import type { CaptureSession, CaptureSessionStatus, CompleteCaptureSessionInput, StartCaptureSessionInput } from "../types/session.js";
+import { SessionStore } from "./session-store.js";
+export declare class SessionManager {
+    private readonly store;
+    private readonly logger;
+    constructor(store: SessionStore, logger: Logger);
+    startSession(input: StartCaptureSessionInput): CaptureSession;
+    activateSession(sessionId: string): CaptureSession;
+    completeSession(input: CompleteCaptureSessionInput): CaptureSession;
+    cancelSession(sessionId: string): CaptureSession;
+    failSession(sessionId: string, errorMessage: string): CaptureSession;
+    getSession(sessionId: string): CaptureSession;
+    listSessions(): CaptureSession[];
+    isTerminalStatus(status: CaptureSessionStatus): boolean;
+    reapTerminalSessions(maxAgeMs: number): {
+        removedSessionIds: string[];
+    };
+    private requireNonTerminalSession;
+    private assertFresh;
+    private assertNotExpired;
+    private isExpired;
+    private expireSession;
+    private persistWithStatus;
+    private persist;
+    private requireSession;
+}
+export declare const createMockCaptureBundle: (sessionId: string, command: CaptureBundle["command"]) => CaptureBundle;

package/dist/session/session-manager.js

@@ -0,0 +1,217 @@
+import { randomUUID } from "node:crypto";
+import { AppError } from "../errors/app-error.js";
+const DEFAULT_SESSION_TTL_MS = 5 * 60 * 1000;
+const TERMINAL_STATUSES = new Set([
+    "completed",
+    "cancelled",
+    "expired",
+    "failed"
+]);
+export class SessionManager {
+    store;
+    logger;
+    constructor(store, logger) {
+        this.store = store;
+        this.logger = logger;
+    }
+    startSession(input) {
+        const now = new Date();
+        const ttlMs = input.ttlMs ?? DEFAULT_SESSION_TTL_MS;
+        const session = {
+            id: randomUUID(),
+            command: input.command,
+            status: "created",
+            createdAt: now.toISOString(),
+            updatedAt: now.toISOString(),
+            expiresAt: new Date(now.getTime() + ttlMs).toISOString()
+        };
+        this.store.set(session);
+        this.logger.info("Started capture session", {
+            sessionId: session.id,
+            command: session.command,
+            expiresAt: session.expiresAt
+        });
+        return session;
+    }
+    activateSession(sessionId) {
+        const session = this.requireNonTerminalSession(sessionId);
+        return this.persistWithStatus(session, "active", "Activated capture session");
+    }
+    completeSession(input) {
+        const session = this.requireNonTerminalSession(input.sessionId);
+        const updated = this.persist({
+            ...session,
+            status: "completed",
+            result: input.bundle,
+            errorMessage: undefined
+        });
+        this.logger.info("Completed capture session", {
+            sessionId: updated.id
+        });
+        return updated;
+    }
+    cancelSession(sessionId) {
+        const session = this.requireSession(sessionId);
+        if (session.status === "completed") {
+            throw new AppError("SESSION_CONFLICT", "Completed sessions cannot be cancelled", {
+                sessionId: session.id
+            });
+        }
+        if (this.isTerminalStatus(session.status)) {
+            return session;
+        }
+        return this.persistWithStatus(session, "cancelled", "Cancelled capture session");
+    }
+    failSession(sessionId, errorMessage) {
+        const session = this.requireSession(sessionId);
+        if (this.isTerminalStatus(session.status)) {
+            return session;
+        }
+        const updated = this.persist({
+            ...session,
+            status: "failed",
+            errorMessage
+        });
+        this.logger.error("Failed capture session", {
+            sessionId: updated.id,
+            errorMessage
+        });
+        return updated;
+    }
+    getSession(sessionId) {
+        const session = this.requireSession(sessionId);
+        return this.assertFresh(session);
+    }
+    listSessions() {
+        return this.store.list().map((session) => this.assertFresh(session));
+    }
+    isTerminalStatus(status) {
+        return TERMINAL_STATUSES.has(status);
+    }
+    reapTerminalSessions(maxAgeMs) {
+        const now = Date.now();
+        const removedSessionIds = [];
+        for (const session of this.listSessions()) {
+            if (!this.isTerminalStatus(session.status)) {
+                continue;
+            }
+            const updatedAtMs = Date.parse(session.updatedAt);
+            if (now - updatedAtMs < maxAgeMs) {
+                continue;
+            }
+            this.store.delete(session.id);
+            removedSessionIds.push(session.id);
+        }
+        if (removedSessionIds.length > 0) {
+            this.logger.info("Reaped terminal capture sessions", {
+                removedSessionIds,
+                maxAgeMs
+            });
+        }
+        return { removedSessionIds };
+    }
+    requireNonTerminalSession(sessionId) {
+        const session = this.requireSession(sessionId);
+        this.assertNotExpired(session);
+        if (this.isTerminalStatus(session.status)) {
+            throw new AppError("SESSION_CONFLICT", "Session cannot transition from its current state", {
+                sessionId: session.id,
+                status: session.status
+            });
+        }
+        return session;
+    }
+    assertFresh(session) {
+        return this.isExpired(session) ? this.expireSession(session) : session;
+    }
+    assertNotExpired(session) {
+        if (this.isExpired(session)) {
+            this.expireSession(session);
+            throw new AppError("SESSION_EXPIRED", "Session has expired", {
+                sessionId: session.id
+            });
+        }
+    }
+    isExpired(session) {
+        return Date.now() > Date.parse(session.expiresAt);
+    }
+    expireSession(session) {
+        if (session.status === "expired") {
+            return session;
+        }
+        const updated = this.persist({
+            ...session,
+            status: "expired"
+        });
+        this.logger.warn("Expired capture session", {
+            sessionId: updated.id
+        });
+        return updated;
+    }
+    persistWithStatus(session, status, logMessage) {
+        const updated = this.persist({
+            ...session,
+            status
+        });
+        this.logger.debug(logMessage, {
+            sessionId: updated.id,
+            status: updated.status
+        });
+        return updated;
+    }
+    persist(session) {
+        const now = new Date().toISOString();
+        const updated = {
+            ...session,
+            updatedAt: now
+        };
+        this.store.set(updated);
+        return updated;
+    }
+    requireSession(sessionId) {
+        const session = this.store.get(sessionId);
+        if (!session) {
+            throw new AppError("NOT_FOUND", "Capture session not found", {
+                sessionId
+            });
+        }
+        return session;
+    }
+}
+export const createMockCaptureBundle = (sessionId, command) => ({
+    sessionId,
+    command,
+    image: {
+        mimeType: "image/png",
+        bytesBase64: "cGg3LW1vY2staW1hZ2U=",
+        width: 1280,
+        height: 720,
+        byteLength: 14,
+        sourceWidth: 1920,
+        sourceHeight: 1080,
+        backend: "mock-capture-bundle",
+        persisted: false
+    },
+    selection: {
+        x: 120,
+        y: 96,
+        width: 640,
+        height: 360
+    },
+    annotations: [
+        {
+            type: "rect",
+            x: 150,
+            y: 120,
+            width: 280,
+            height: 96,
+            label: "Mock highlighted region"
+        }
+    ],
+    context: {
+        activeAppName: "Mock Browser",
+        activeWindowTitle: "Phase 7 Prototype",
+        capturedAt: new Date().toISOString(),
+        displayId: "display-1"
+    }
+});

package/dist/session/session-store.d.ts

@@ -0,0 +1,8 @@
+import type { CaptureSession } from "../types/session.js";
+export declare class SessionStore {
+    private readonly sessions;
+    get(sessionId: string): CaptureSession | undefined;
+    set(session: CaptureSession): void;
+    list(): CaptureSession[];
+    delete(sessionId: string): boolean;
+}

package/dist/session/session-store.js

@@ -0,0 +1,15 @@
+export class SessionStore {
+    sessions = new Map();
+    get(sessionId) {
+        return this.sessions.get(sessionId);
+    }
+    set(session) {
+        this.sessions.set(session.id, session);
+    }
+    list() {
+        return Array.from(this.sessions.values()).sort((left, right) => left.createdAt.localeCompare(right.createdAt));
+    }
+    delete(sessionId) {
+        return this.sessions.delete(sessionId);
+    }
+}

package/dist/session/session-waiter.d.ts

@@ -0,0 +1,14 @@
+import type { Logger } from "../logging/logger.js";
+import type { AwaitCaptureSessionInput, AwaitCaptureSessionResult, CaptureSession } from "../types/session.js";
+import { SessionManager } from "./session-manager.js";
+export declare class SessionWaiter {
+    private readonly sessionManager;
+    private readonly logger;
+    private readonly pendingWaits;
+    constructor(sessionManager: SessionManager, logger: Logger);
+    awaitSession(input: AwaitCaptureSessionInput): Promise<AwaitCaptureSessionResult>;
+    notify(session: CaptureSession): void;
+    clearSession(sessionId: string): void;
+    private toTerminalOutcome;
+    private removeWait;
+}

package/dist/session/session-waiter.js

@@ -0,0 +1,102 @@
+export class SessionWaiter {
+    sessionManager;
+    logger;
+    pendingWaits = new Map();
+    constructor(sessionManager, logger) {
+        this.sessionManager = sessionManager;
+        this.logger = logger;
+    }
+    awaitSession(input) {
+        const session = this.sessionManager.getSession(input.sessionId);
+        const timeoutMs = input.timeoutMs ?? 30_000;
+        if (this.sessionManager.isTerminalStatus(session.status)) {
+            return Promise.resolve(this.toTerminalOutcome(session));
+        }
+        const expiresInMs = Math.max(0, Date.parse(session.expiresAt) - Date.now());
+        const waitMs = Math.min(timeoutMs, expiresInMs || timeoutMs);
+        return new Promise((resolve) => {
+            const pendingWait = {
+                resolve,
+                timer: setTimeout(() => {
+                    this.removeWait(session.id, pendingWait);
+                    const currentSession = this.sessionManager.getSession(session.id);
+                    if (this.sessionManager.isTerminalStatus(currentSession.status)) {
+                        resolve(this.toTerminalOutcome(currentSession));
+                        return;
+                    }
+                    resolve({
+                        outcome: "timed_out",
+                        session: currentSession,
+                        waitedMs: timeoutMs
+                    });
+                }, waitMs)
+            };
+            const pending = this.pendingWaits.get(session.id) ?? new Set();
+            pending.add(pendingWait);
+            this.pendingWaits.set(session.id, pending);
+            this.logger.debug("Registered capture session wait", {
+                sessionId: session.id,
+                timeoutMs,
+                waitMs
+            });
+        });
+    }
+    notify(session) {
+        if (!this.sessionManager.isTerminalStatus(session.status)) {
+            return;
+        }
+        const pending = this.pendingWaits.get(session.id);
+        if (!pending || pending.size === 0) {
+            return;
+        }
+        const outcome = this.toTerminalOutcome(session);
+        for (const wait of pending) {
+            clearTimeout(wait.timer);
+            wait.resolve(outcome);
+        }
+        this.pendingWaits.delete(session.id);
+        this.logger.debug("Resolved capture session waits", {
+            sessionId: session.id,
+            outcome: outcome.outcome
+        });
+    }
+    clearSession(sessionId) {
+        const pending = this.pendingWaits.get(sessionId);
+        if (!pending) {
+            return;
+        }
+        for (const wait of pending) {
+            clearTimeout(wait.timer);
+        }
+        this.pendingWaits.delete(sessionId);
+        this.logger.debug("Cleared capture session waits", {
+            sessionId
+        });
+    }
+    toTerminalOutcome(session) {
+        if (session.status === "completed" && session.result) {
+            return {
+                outcome: "completed",
+                session,
+                result: session.result
+            };
+        }
+        if (session.status === "cancelled" || session.status === "expired" || session.status === "failed") {
+            return {
+                outcome: session.status,
+                session
+            };
+        }
+        throw new Error(`Expected terminal capture session status, received ${session.status}`);
+    }
+    removeWait(sessionId, pendingWait) {
+        const pending = this.pendingWaits.get(sessionId);
+        if (!pending) {
+            return;
+        }
+        pending.delete(pendingWait);
+        if (pending.size === 0) {
+            this.pendingWaits.delete(sessionId);
+        }
+    }
+}

package/dist/types/annotation.d.ts

@@ -0,0 +1,32 @@
+export type Point = {
+    x: number;
+    y: number;
+};
+export type RectangleAnnotation = {
+    type: "rect";
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    label?: string;
+};
+export type ArrowAnnotation = {
+    type: "arrow";
+    from: Point;
+    to: Point;
+    label?: string;
+};
+export type TextAnnotation = {
+    type: "text";
+    x: number;
+    y: number;
+    text: string;
+};
+export type RedactAnnotation = {
+    type: "redact";
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+};
+export type Annotation = RectangleAnnotation | ArrowAnnotation | TextAnnotation | RedactAnnotation;

package/dist/types/annotation.js

@@ -0,0 +1 @@
+export {};

package/dist/types/capture.d.ts

@@ -0,0 +1,33 @@
+import type { Annotation } from "./annotation.js";
+export type CaptureCommand = "see" | "clip";
+export type SelectionBounds = {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+};
+export type CaptureImage = {
+    mimeType: "image/png";
+    bytesBase64: string;
+    width: number;
+    height: number;
+    byteLength: number;
+    sourceWidth: number;
+    sourceHeight: number;
+    backend: string;
+    persisted: false;
+};
+export type CaptureContext = {
+    activeAppName?: string;
+    activeWindowTitle?: string;
+    capturedAt: string;
+    displayId?: string;
+};
+export type CaptureBundle = {
+    sessionId: string;
+    command: CaptureCommand;
+    image: CaptureImage;
+    selection: SelectionBounds;
+    annotations: Annotation[];
+    context?: CaptureContext;
+};