agent-vision-mcp 0.1.0

package/dist/overlay/local-overlay-agent.js

@@ -0,0 +1,551 @@
+import { randomUUID } from "node:crypto";
+import { AppError } from "../errors/app-error.js";
+const TERMINAL_STATUSES = new Set([
+    "sent",
+    "cancelled",
+    "failed",
+    "expired"
+]);
+const OVERLAY_SHORTCUTS = {
+    select: "V",
+    rect: "B",
+    arrow: "A",
+    text: "T",
+    redact: "R"
+};
+const TOOL_LABELS = {
+    select: "Select",
+    rect: "Box",
+    arrow: "Arrow",
+    text: "Text",
+    redact: "Redact"
+};
+const TOOL_DESCRIPTIONS = {
+    select: "Create or adjust the capture region.",
+    rect: "Highlight an area with a rectangle.",
+    arrow: "Point at a detail that needs attention.",
+    text: "Add a short note for the model.",
+    redact: "Hide sensitive content before sending."
+};
+const OVERLAY_TERMINAL_BY_CAPTURE_STATUS = {
+    completed: "sent",
+    cancelled: "cancelled",
+    failed: "failed",
+    expired: "expired"
+};
+const TOOL_ORDER = ["select", "rect", "arrow", "text", "redact"];
+const buildToolDescriptors = () => TOOL_ORDER.map((tool) => ({
+    tool,
+    label: TOOL_LABELS[tool],
+    shortcut: OVERLAY_SHORTCUTS[tool],
+    description: TOOL_DESCRIPTIONS[tool]
+}));
+export class LocalOverlayAgent {
+    captureSessions;
+    capturePipeline;
+    logger;
+    overlaySessions = new Map();
+    constructor(captureSessions, capturePipeline, logger) {
+        this.captureSessions = captureSessions;
+        this.capturePipeline = capturePipeline;
+        this.logger = logger;
+    }
+    launch(sessionId) {
+        const captureSession = this.captureSessions.getSession(sessionId);
+        const existing = this.overlaySessions.get(sessionId);
+        if (existing && !this.isTerminal(existing.status)) {
+            throw new AppError("SESSION_CONFLICT", "Overlay session is already active", { sessionId });
+        }
+        const now = new Date().toISOString();
+        const overlaySession = this.present({
+            sessionId: captureSession.id,
+            command: captureSession.command,
+            status: "armed",
+            createdAt: now,
+            updatedAt: now,
+            activeTool: "select",
+            annotations: [],
+            shortcuts: OVERLAY_SHORTCUTS,
+            toolDescriptors: buildToolDescriptors(),
+            selectionSummary: { label: "No region selected yet", areaPx: 0 },
+            annotationSummary: { total: 0, byType: {} },
+            preview: {
+                canSend: false,
+                showSelectionOutline: false,
+                showAnnotationLayer: false,
+                emptyStateMessage: "Select a region to start the preview."
+            },
+            hud: {
+                title: "Capture Armed",
+                subtitle: "Switch to the target screen and drag a region to begin.",
+                statusBadge: "Waiting for selection",
+                primaryActionLabel: "Select Region",
+                secondaryActionLabel: "Cancel",
+                activeToolLabel: "Select (V)",
+                onboardingHint: "Tip: start with a tight crop, then annotate only what matters."
+            }
+        });
+        this.overlaySessions.set(sessionId, overlaySession);
+        this.logger.info("Launched overlay capture session", { sessionId });
+        return overlaySession;
+    }
+    get(sessionId) {
+        return this.syncWithCaptureSession(this.requireOverlaySession(sessionId));
+    }
+    list() {
+        return Array.from(this.overlaySessions.values())
+            .map((session) => this.syncWithCaptureSession(session))
+            .sort((left, right) => left.createdAt.localeCompare(right.createdAt));
+    }
+    setActiveTool(sessionId, tool) {
+        const session = this.requireMutableOverlaySession(sessionId);
+        const updated = this.persist({
+            ...session,
+            activeTool: tool
+        });
+        this.logger.debug("Set overlay active tool", { sessionId, tool });
+        return updated;
+    }
+    selectRegion(sessionId, input) {
+        this.assertValidBounds(input);
+        const session = this.requireMutableOverlaySession(sessionId);
+        const updated = this.persist({
+            ...session,
+            status: "selected",
+            selection: {
+                x: input.x,
+                y: input.y,
+                width: input.width,
+                height: input.height
+            },
+            context: {
+                displayId: input.displayId,
+                activeAppName: input.activeAppName,
+                activeWindowTitle: input.activeWindowTitle
+            }
+        });
+        this.logger.debug("Selected overlay region", { sessionId, selection: updated.selection });
+        return updated;
+    }
+    moveSelection(sessionId, input) {
+        const session = this.requireSelectedOverlaySession(sessionId);
+        const selection = session.selection;
+        const updated = this.persist({
+            ...session,
+            selection: {
+                ...selection,
+                x: selection.x + input.dx,
+                y: selection.y + input.dy
+            }
+        });
+        this.logger.debug("Moved overlay selection", { sessionId, selection: updated.selection });
+        return updated;
+    }
+    resizeSelection(sessionId, input) {
+        const session = this.requireSelectedOverlaySession(sessionId);
+        const selection = session.selection;
+        const nextSelection = {
+            x: input.x ?? selection.x,
+            y: input.y ?? selection.y,
+            width: input.width ?? selection.width,
+            height: input.height ?? selection.height
+        };
+        this.assertValidBounds(nextSelection);
+        const updated = this.persist({
+            ...session,
+            selection: nextSelection
+        });
+        this.logger.debug("Resized overlay selection", { sessionId, selection: updated.selection });
+        return updated;
+    }
+    addAnnotation(sessionId, input) {
+        const session = this.requireSelectedOverlaySession(sessionId);
+        this.assertAnnotation(input.annotation);
+        const now = new Date().toISOString();
+        const record = {
+            id: input.id ?? randomUUID(),
+            annotation: input.annotation,
+            createdAt: now,
+            updatedAt: now
+        };
+        const updated = this.persist({
+            ...session,
+            annotations: [...session.annotations, record]
+        });
+        this.logger.debug("Added overlay annotation", {
+            sessionId,
+            annotationId: record.id,
+            annotationType: record.annotation.type
+        });
+        return updated;
+    }
+    updateAnnotation(sessionId, input) {
+        const session = this.requireSelectedOverlaySession(sessionId);
+        this.assertAnnotation(input.annotation);
+        const existing = session.annotations.find((annotation) => annotation.id === input.annotationId);
+        if (!existing) {
+            throw new AppError("NOT_FOUND", "Overlay annotation not found", {
+                sessionId,
+                annotationId: input.annotationId
+            });
+        }
+        const updated = this.persist({
+            ...session,
+            annotations: session.annotations.map((record) => record.id === input.annotationId
+                ? {
+                    ...record,
+                    annotation: input.annotation,
+                    updatedAt: new Date().toISOString()
+                }
+                : record)
+        });
+        this.logger.debug("Updated overlay annotation", {
+            sessionId,
+            annotationId: input.annotationId,
+            annotationType: input.annotation.type
+        });
+        return updated;
+    }
+    removeAnnotation(sessionId, annotationId) {
+        const session = this.requireSelectedOverlaySession(sessionId);
+        const annotationExists = session.annotations.some((record) => record.id === annotationId);
+        if (!annotationExists) {
+            throw new AppError("NOT_FOUND", "Overlay annotation not found", {
+                sessionId,
+                annotationId
+            });
+        }
+        const updated = this.persist({
+            ...session,
+            annotations: session.annotations.filter((record) => record.id !== annotationId)
+        });
+        this.logger.debug("Removed overlay annotation", { sessionId, annotationId });
+        return updated;
+    }
+    clearAnnotations(sessionId) {
+        const session = this.requireSelectedOverlaySession(sessionId);
+        const updated = this.persist({
+            ...session,
+            annotations: []
+        });
+        this.logger.debug("Cleared overlay annotations", { sessionId });
+        return updated;
+    }
+    async send(sessionId) {
+        const overlaySession = this.requireSelectedOverlaySession(sessionId);
+        try {
+            const bundle = await this.capturePipeline.createBundle({
+                sessionId,
+                command: overlaySession.command,
+                selection: overlaySession.selection,
+                annotations: overlaySession.annotations.map((entry) => entry.annotation),
+                context: overlaySession.context
+            });
+            const completed = this.captureSessions.completeSession({
+                sessionId,
+                bundle
+            });
+            this.persist({
+                ...overlaySession,
+                status: "sent",
+                errorMessage: undefined
+            });
+            this.logger.info("Sent overlay capture session", {
+                sessionId,
+                annotationCount: overlaySession.annotations.length,
+                backend: bundle.image.backend,
+                byteLength: bundle.image.byteLength
+            });
+            return completed;
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            const failed = this.fail(sessionId, `Overlay send failed: ${message}`);
+            throw new AppError("INTERNAL_ERROR", "Overlay capture send failed", {
+                sessionId,
+                cause: message,
+                resultingStatus: failed.status
+            });
+        }
+    }
+    cancel(sessionId) {
+        const overlaySession = this.syncWithCaptureSession(this.requireOverlaySession(sessionId));
+        if (!this.isTerminal(overlaySession.status)) {
+            this.persist({
+                ...overlaySession,
+                status: "cancelled"
+            });
+        }
+        const cancelled = this.captureSessions.cancelSession(sessionId);
+        this.logger.info("Cancelled overlay capture session", { sessionId });
+        return cancelled;
+    }
+    fail(sessionId, errorMessage) {
+        const overlaySession = this.syncWithCaptureSession(this.requireOverlaySession(sessionId));
+        if (!this.isTerminal(overlaySession.status)) {
+            this.persist({
+                ...overlaySession,
+                status: "failed",
+                errorMessage
+            });
+        }
+        const failed = this.captureSessions.failSession(sessionId, errorMessage);
+        this.logger.error("Failed overlay capture session", { sessionId, errorMessage });
+        return failed;
+    }
+    reapTerminalSessions(maxAgeMs) {
+        const now = Date.now();
+        const removedSessionIds = [];
+        for (const session of this.list()) {
+            if (!this.isTerminal(session.status)) {
+                continue;
+            }
+            const updatedAtMs = Date.parse(session.updatedAt);
+            if (now - updatedAtMs < maxAgeMs) {
+                continue;
+            }
+            this.overlaySessions.delete(session.sessionId);
+            removedSessionIds.push(session.sessionId);
+        }
+        if (removedSessionIds.length > 0) {
+            this.logger.info("Reaped terminal overlay sessions", {
+                removedSessionIds,
+                maxAgeMs
+            });
+        }
+        return { removedSessionIds };
+    }
+    requireSelectedOverlaySession(sessionId) {
+        const session = this.requireMutableOverlaySession(sessionId);
+        if (!session.selection) {
+            throw new AppError("SESSION_CONFLICT", "Overlay session does not have a selected region", {
+                sessionId
+            });
+        }
+        return session;
+    }
+    requireMutableOverlaySession(sessionId) {
+        const session = this.syncWithCaptureSession(this.requireOverlaySession(sessionId));
+        if (this.isTerminal(session.status)) {
+            throw new AppError("SESSION_CONFLICT", "Overlay session is no longer mutable", {
+                sessionId,
+                status: session.status
+            });
+        }
+        return session;
+    }
+    requireOverlaySession(sessionId) {
+        const session = this.overlaySessions.get(sessionId);
+        if (!session) {
+            throw new AppError("NOT_FOUND", "Overlay session not found", { sessionId });
+        }
+        return session;
+    }
+    syncWithCaptureSession(session) {
+        let captureSession;
+        try {
+            captureSession = this.captureSessions.getSession(session.sessionId);
+        }
+        catch {
+            return session;
+        }
+        const mappedStatus = OVERLAY_TERMINAL_BY_CAPTURE_STATUS[captureSession.status];
+        if (!mappedStatus || session.status === mappedStatus) {
+            return session;
+        }
+        const updated = this.persist({
+            ...session,
+            status: mappedStatus,
+            errorMessage: captureSession.errorMessage ?? session.errorMessage
+        });
+        this.logger.debug("Synchronized overlay session with capture session", {
+            sessionId: session.sessionId,
+            captureStatus: captureSession.status,
+            overlayStatus: updated.status
+        });
+        return updated;
+    }
+    persist(session) {
+        const updated = this.present({
+            ...session,
+            updatedAt: new Date().toISOString()
+        });
+        this.overlaySessions.set(updated.sessionId, updated);
+        return updated;
+    }
+    present(session) {
+        const selectionSummary = this.buildSelectionSummary(session.selection);
+        const annotationSummary = this.buildAnnotationSummary(session.annotations);
+        const preview = this.buildPreviewState(session, selectionSummary, annotationSummary);
+        const hud = this.buildHudState(session, selectionSummary, annotationSummary, preview);
+        return {
+            ...session,
+            toolDescriptors: buildToolDescriptors(),
+            selectionSummary,
+            annotationSummary,
+            preview,
+            hud
+        };
+    }
+    buildSelectionSummary(selection) {
+        if (!selection) {
+            return {
+                label: "No region selected yet",
+                areaPx: 0
+            };
+        }
+        const areaPx = selection.width * selection.height;
+        return {
+            label: `${selection.width} x ${selection.height} at (${selection.x}, ${selection.y})`,
+            areaPx,
+            bounds: selection
+        };
+    }
+    buildAnnotationSummary(annotations) {
+        const byType = {};
+        for (const entry of annotations) {
+            byType[entry.annotation.type] = (byType[entry.annotation.type] ?? 0) + 1;
+        }
+        return {
+            total: annotations.length,
+            byType,
+            latestAnnotationId: annotations.at(-1)?.id
+        };
+    }
+    buildPreviewState(session, selectionSummary, annotationSummary) {
+        const hasSelection = Boolean(session.selection);
+        const canSend = hasSelection && !this.isTerminal(session.status);
+        return {
+            canSend,
+            showSelectionOutline: hasSelection,
+            showAnnotationLayer: annotationSummary.total > 0,
+            emptyStateMessage: hasSelection
+                ? undefined
+                : "Select a region to activate the preview and annotation layer."
+        };
+    }
+    buildHudState(session, selectionSummary, annotationSummary, preview) {
+        const activeToolLabel = `${TOOL_LABELS[session.activeTool]} (${session.shortcuts[session.activeTool]})`;
+        if (session.status === "armed") {
+            return {
+                title: "Capture Armed",
+                subtitle: "Switch to the target screen and drag a region to begin.",
+                statusBadge: "Waiting for selection",
+                primaryActionLabel: "Select Region",
+                secondaryActionLabel: "Cancel",
+                activeToolLabel,
+                onboardingHint: "Start with a focused crop. You can annotate after selecting."
+            };
+        }
+        if (session.status === "selected") {
+            return {
+                title: annotationSummary.total > 0 ? "Preview Ready" : "Selection Ready",
+                subtitle: annotationSummary.total > 0
+                    ? `${annotationSummary.total} annotation${annotationSummary.total === 1 ? "" : "s"} added.`
+                    : "Add markup or send the current selection as-is.",
+                statusBadge: preview.canSend ? "Ready to send" : "Preview unavailable",
+                primaryActionLabel: preview.canSend ? "Send Capture" : "Select Region",
+                secondaryActionLabel: annotationSummary.total > 0 ? "Clear Annotations" : "Cancel",
+                activeToolLabel,
+                onboardingHint: `Selection: ${selectionSummary.label}`
+            };
+        }
+        if (session.status === "sent") {
+            return {
+                title: "Capture Sent",
+                subtitle: "The capture bundle has been returned to the LLM workflow.",
+                statusBadge: "Delivered",
+                primaryActionLabel: "Done",
+                secondaryActionLabel: "Close",
+                activeToolLabel,
+                onboardingHint: "Nothing was persisted by default."
+            };
+        }
+        if (session.status === "failed") {
+            return {
+                title: "Capture Failed",
+                subtitle: session.errorMessage ?? "The overlay session failed before completion.",
+                statusBadge: "Error",
+                primaryActionLabel: "Retry",
+                secondaryActionLabel: "Close",
+                activeToolLabel,
+                onboardingHint: "Inspect the error and start a new capture session if needed."
+            };
+        }
+        if (session.status === "expired") {
+            return {
+                title: "Capture Expired",
+                subtitle: "The session timed out before send.",
+                statusBadge: "Expired",
+                primaryActionLabel: "Retry",
+                secondaryActionLabel: "Close",
+                activeToolLabel,
+                onboardingHint: "Start a new session to continue."
+            };
+        }
+        return {
+            title: "Capture Cancelled",
+            subtitle: "The overlay session was cancelled.",
+            statusBadge: "Cancelled",
+            primaryActionLabel: "Retry",
+            secondaryActionLabel: "Close",
+            activeToolLabel,
+            onboardingHint: "You can start a new session whenever you are ready."
+        };
+    }
+    assertValidBounds(bounds) {
+        if (bounds.width <= 0 || bounds.height <= 0) {
+            throw new AppError("INVALID_ARGUMENT", "Selection width and height must be greater than zero", {
+                bounds
+            });
+        }
+    }
+    assertAnnotation(annotation) {
+        switch (annotation.type) {
+            case "rect":
+                this.assertBoxAnnotation(annotation);
+                return;
+            case "redact":
+                this.assertRedactAnnotation(annotation);
+                return;
+            case "arrow":
+                this.assertArrowAnnotation(annotation);
+                return;
+            case "text":
+                this.assertTextAnnotation(annotation);
+                return;
+            default:
+                throw new AppError("INVALID_ARGUMENT", "Unsupported annotation type", {
+                    annotation
+                });
+        }
+    }
+    assertBoxAnnotation(annotation) {
+        this.assertValidBounds(annotation);
+    }
+    assertRedactAnnotation(annotation) {
+        this.assertValidBounds(annotation);
+    }
+    assertArrowAnnotation(annotation) {
+        if (Number.isNaN(annotation.from.x) || Number.isNaN(annotation.from.y)) {
+            throw new AppError("INVALID_ARGUMENT", "Arrow annotation start point must be numeric", {
+                annotation
+            });
+        }
+        if (Number.isNaN(annotation.to.x) || Number.isNaN(annotation.to.y)) {
+            throw new AppError("INVALID_ARGUMENT", "Arrow annotation end point must be numeric", {
+                annotation
+            });
+        }
+    }
+    assertTextAnnotation(annotation) {
+        if (annotation.text.trim() === "") {
+            throw new AppError("INVALID_ARGUMENT", "Text annotation must include non-empty text", {
+                annotation
+            });
+        }
+    }
+    isTerminal(status) {
+        return TERMINAL_STATUSES.has(status);
+    }
+}

package/dist/overlay/overlay-bundle-factory.d.ts

@@ -0,0 +1,4 @@
+import type { CaptureBundle } from "../types/capture.js";
+import type { CaptureSession } from "../types/session.js";
+import type { OverlaySession } from "./types.js";
+export declare const createOverlayCaptureBundle: (session: CaptureSession, overlaySession: OverlaySession) => CaptureBundle;

package/dist/overlay/overlay-bundle-factory.js

@@ -0,0 +1,24 @@
+export const createOverlayCaptureBundle = (session, overlaySession) => {
+    if (!overlaySession.selection) {
+        throw new Error("Cannot build capture bundle without a selection");
+    }
+    const selection = overlaySession.selection;
+    return {
+        sessionId: session.id,
+        command: session.command,
+        image: {
+            mimeType: "image/png",
+            bytesBase64: "cGg0LW92ZXJsYXktYW5ub3RhdGVkLWltYWdl",
+            width: Math.max(selection.width, 1),
+            height: Math.max(selection.height, 1)
+        },
+        selection,
+        annotations: overlaySession.annotations.map((entry) => entry.annotation),
+        context: {
+            activeAppName: overlaySession.context?.activeAppName ?? "Prototype App",
+            activeWindowTitle: overlaySession.context?.activeWindowTitle ?? "Overlay Prototype Window",
+            capturedAt: new Date().toISOString(),
+            displayId: overlaySession.context?.displayId ?? "display-1"
+        }
+    };
+};

package/dist/overlay/types.d.ts

@@ -0,0 +1,83 @@
+import type { Annotation } from "../types/annotation.js";
+import type { CaptureCommand, SelectionBounds } from "../types/capture.js";
+export type OverlaySessionStatus = "armed" | "selected" | "sent" | "cancelled" | "failed" | "expired";
+export type OverlayTool = "select" | "rect" | "arrow" | "text" | "redact";
+export type OverlaySelectionContext = {
+    displayId?: string;
+    activeAppName?: string;
+    activeWindowTitle?: string;
+};
+export type OverlayAnnotationRecord = {
+    id: string;
+    annotation: Annotation;
+    createdAt: string;
+    updatedAt: string;
+};
+export type OverlayToolDescriptor = {
+    tool: OverlayTool;
+    label: string;
+    shortcut: string;
+    description: string;
+};
+export type OverlaySelectionSummary = {
+    label: string;
+    areaPx: number;
+    bounds?: SelectionBounds;
+};
+export type OverlayAnnotationSummary = {
+    total: number;
+    byType: Partial<Record<Annotation["type"], number>>;
+    latestAnnotationId?: string;
+};
+export type OverlayPreviewState = {
+    canSend: boolean;
+    showSelectionOutline: boolean;
+    showAnnotationLayer: boolean;
+    emptyStateMessage?: string;
+};
+export type OverlayHudState = {
+    title: string;
+    subtitle: string;
+    statusBadge: string;
+    primaryActionLabel: string;
+    secondaryActionLabel: string;
+    activeToolLabel: string;
+    onboardingHint: string;
+};
+export type OverlaySession = {
+    sessionId: string;
+    command: CaptureCommand;
+    status: OverlaySessionStatus;
+    createdAt: string;
+    updatedAt: string;
+    selection?: SelectionBounds;
+    context?: OverlaySelectionContext;
+    activeTool: OverlayTool;
+    annotations: OverlayAnnotationRecord[];
+    shortcuts: Record<OverlayTool, string>;
+    toolDescriptors: OverlayToolDescriptor[];
+    selectionSummary: OverlaySelectionSummary;
+    annotationSummary: OverlayAnnotationSummary;
+    preview: OverlayPreviewState;
+    hud: OverlayHudState;
+    errorMessage?: string;
+};
+export type OverlayRegionInput = SelectionBounds & OverlaySelectionContext;
+export type OverlayMoveInput = {
+    dx: number;
+    dy: number;
+};
+export type OverlayResizeInput = {
+    x?: number;
+    y?: number;
+    width?: number;
+    height?: number;
+};
+export type CreateAnnotationInput = {
+    id?: string;
+    annotation: Annotation;
+};
+export type UpdateAnnotationInput = {
+    annotationId: string;
+    annotation: Annotation;
+};

package/dist/overlay/types.js

@@ -0,0 +1 @@
+export {};

package/dist/server.d.ts

@@ -0,0 +1,19 @@
+import { ConsoleLogger } from "./logging/logger.js";
+import { ToolRegistry } from "./mcp/tool-registry.js";
+export declare class VisualContextServer {
+    readonly logger: ConsoleLogger;
+    private readonly cdpClient;
+    private readonly cdpDiscovery;
+    private readonly liveTabs;
+    private readonly browserLiveTabs;
+    private readonly tabResolution;
+    private readonly tabScreenshots;
+    private readonly tabContext;
+    private readonly browserSee;
+    private readonly tools;
+    constructor();
+    listTools(): ReturnType<ToolRegistry["list"]>;
+    callTool(name: string, args?: Record<string, unknown>): Promise<unknown>;
+    start(): void;
+    private registerTools;
+}