agent-vision-mcp 0.1.0

package/dist/desktop/main.js

@@ -0,0 +1,80 @@
+import { app, globalShortcut, screen } from "electron";
+import { OverlayDesktopController } from "./controller.js";
+const controller = new OverlayDesktopController();
+const captureShortcut = "CommandOrControl+Shift+2";
+// Electron's Wayland docs require Chromium's GlobalShortcutsPortal
+// for globalShortcut support in Wayland sessions.
+app.commandLine.appendSwitch("enable-features", "GlobalShortcutsPortal");
+const getDefaultDisplaySelection = () => {
+    const display = screen.getPrimaryDisplay();
+    const { x, y, width, height } = display.workArea;
+    return {
+        x,
+        y,
+        width,
+        height,
+        displayId: String(display.id),
+        activeAppName: "Current Display",
+        activeWindowTitle: "Wayland portal capture"
+    };
+};
+const triggerActiveWindowCapture = async (payload) => {
+    const capturePayload = {
+        command: payload?.command ?? "see",
+        selection: payload?.selection ?? getDefaultDisplaySelection()
+    };
+    console.log("[desktop] capture requested", {
+        shortcut: captureShortcut,
+        command: capturePayload.command,
+        selection: capturePayload.selection
+    });
+    const result = await controller.captureActiveWindow(capturePayload);
+    console.log("[desktop] capture completed", {
+        stage: result.stage,
+        outcome: result.waitResult.outcome,
+        sessionId: result.sessionId
+    });
+    return result;
+};
+const registerShortcuts = () => {
+    const registered = globalShortcut.register(captureShortcut, () => {
+        console.log("[desktop] hotkey pressed", {
+            shortcut: captureShortcut,
+            at: new Date().toISOString()
+        });
+        void triggerActiveWindowCapture().catch((error) => {
+            console.error("[desktop] active-window capture failed:", {
+                shortcut: captureShortcut,
+                error: String(error?.message ?? error)
+            });
+        });
+    });
+    console.log("[desktop] shortcut registration", {
+        shortcut: captureShortcut,
+        registered,
+        isRegistered: globalShortcut.isRegistered(captureShortcut),
+        sessionType: process.env.XDG_SESSION_TYPE ?? "unknown",
+        waylandDisplay: process.env.WAYLAND_DISPLAY ?? "unset",
+        display: process.env.DISPLAY ?? "unset"
+    });
+    if (!registered) {
+        console.error("[desktop] shortcut registration failed", {
+            shortcut: captureShortcut,
+            hint: "On Wayland, this usually means the compositor or portal rejected the global shortcut."
+        });
+    }
+};
+app.whenReady().then(() => {
+    if (process.platform === "darwin") {
+        app.dock?.hide();
+    }
+    registerShortcuts();
+    console.log("[desktop] background capture hotkey armed", {
+        shortcut: captureShortcut,
+        platform: process.platform
+    });
+});
+app.on("will-quit", () => {
+    console.log("[desktop] unregistering shortcuts");
+    globalShortcut.unregisterAll();
+});

package/dist/desktop/preload.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/desktop/preload.js

@@ -0,0 +1,26 @@
+import { contextBridge, ipcRenderer } from "electron";
+const overlayDesktop = {
+    begin: (payload) => ipcRenderer.invoke("overlay:begin", payload),
+    getStatus: () => ipcRenderer.invoke("overlay:status"),
+    selectRegion: (bounds) => ipcRenderer.invoke("overlay:select-region", bounds),
+    setTool: (tool) => ipcRenderer.invoke("overlay:set-tool", tool),
+    addAnnotation: (annotation) => ipcRenderer.invoke("overlay:add-annotation", annotation),
+    clearAnnotations: () => ipcRenderer.invoke("overlay:clear-annotations"),
+    send: () => ipcRenderer.invoke("overlay:send"),
+    cancel: () => ipcRenderer.invoke("overlay:cancel"),
+    captureActiveWindow: (payload) => ipcRenderer.invoke("overlay:capture-active-window", payload)
+};
+const overlayDesktopEvents = {
+    onRefresh: (listener) => {
+        ipcRenderer.on("control:refresh", () => {
+            listener();
+        });
+    },
+    onError: (listener) => {
+        ipcRenderer.on("control:error", (_event, message) => {
+            listener(message);
+        });
+    }
+};
+contextBridge.exposeInMainWorld("overlayDesktop", overlayDesktop);
+contextBridge.exposeInMainWorld("overlayDesktopEvents", overlayDesktopEvents);

package/dist/desktop/types.d.ts

@@ -0,0 +1,31 @@
+import type { AwaitVisualCaptureResult, BeginVisualCaptureResult, VisualCaptureStatusResult } from "../flow/types.js";
+import type { Annotation } from "../types/annotation.js";
+import type { SelectionBounds } from "../types/capture.js";
+import type { OverlayTool } from "../overlay/types.js";
+export type OverlayBeginPayload = {
+    command?: "see" | "clip";
+};
+export type OverlaySelectRegionPayload = SelectionBounds & {
+    displayId?: string;
+    activeAppName?: string;
+    activeWindowTitle?: string;
+};
+export type ActiveWindowCapturePayload = {
+    command?: "see" | "clip";
+    selection: OverlaySelectRegionPayload;
+};
+export interface OverlayDesktopApi {
+    begin(payload?: OverlayBeginPayload): Promise<BeginVisualCaptureResult>;
+    getStatus(): Promise<VisualCaptureStatusResult>;
+    selectRegion(bounds: OverlaySelectRegionPayload): Promise<VisualCaptureStatusResult>;
+    setTool(tool: OverlayTool): Promise<VisualCaptureStatusResult>;
+    addAnnotation(annotation: Annotation): Promise<VisualCaptureStatusResult>;
+    clearAnnotations(): Promise<VisualCaptureStatusResult>;
+    send(): Promise<AwaitVisualCaptureResult>;
+    cancel(): Promise<VisualCaptureStatusResult>;
+    captureActiveWindow(payload: ActiveWindowCapturePayload): Promise<AwaitVisualCaptureResult>;
+}
+export interface OverlayDesktopEvents {
+    onRefresh(listener: () => void): void;
+    onError(listener: (message: string) => void): void;
+}

package/dist/desktop/types.js

@@ -0,0 +1 @@
+export {};

package/dist/errors/app-error.d.ts

@@ -0,0 +1,7 @@
+export type ErrorCode = "INVALID_ARGUMENT" | "NOT_FOUND" | "SESSION_EXPIRED" | "SESSION_CONFLICT" | "INTERNAL_ERROR";
+export declare class AppError extends Error {
+    readonly code: ErrorCode;
+    readonly details?: Record<string, unknown>;
+    constructor(code: ErrorCode, message: string, details?: Record<string, unknown>);
+}
+export declare const isAppError: (value: unknown) => value is AppError;

package/dist/errors/app-error.js

@@ -0,0 +1,11 @@
+export class AppError extends Error {
+    code;
+    details;
+    constructor(code, message, details) {
+        super(message);
+        this.name = "AppError";
+        this.code = code;
+        this.details = details;
+    }
+}
+export const isAppError = (value) => value instanceof AppError;

package/dist/flow/types.d.ts

@@ -0,0 +1,48 @@
+import type { CaptureBundle, CaptureCommand } from "../types/capture.js";
+import type { OverlaySession } from "../overlay/types.js";
+import type { AwaitCaptureSessionResult, CaptureSession, CaptureSessionStatus } from "../types/session.js";
+export type VisualCaptureStage = "armed" | "selecting" | "annotating" | "ready_to_send" | "completed" | "cancelled" | "failed" | "expired" | "timed_out";
+export type VisualCaptureAction = "switch_to_target" | "select_region" | "annotate" | "send" | "wait" | "done" | "cancel" | "retry";
+export type VisualCaptureGuidance = {
+    title: string;
+    message: string;
+    nextActions: VisualCaptureAction[];
+    primaryAction: VisualCaptureAction;
+    highlightedShortcut?: string;
+    footerHint?: string;
+};
+export type VisualCaptureClientHints = {
+    statusTone: "info" | "success" | "warning" | "error";
+    showToolbar: boolean;
+    showPreview: boolean;
+    allowInlineTips: boolean;
+};
+export type BeginVisualCaptureResult = {
+    sessionId: string;
+    command: CaptureCommand;
+    captureSession: CaptureSession;
+    overlaySession: OverlaySession;
+    stage: VisualCaptureStage;
+    guidance: VisualCaptureGuidance;
+    clientHints: VisualCaptureClientHints;
+};
+export type VisualCaptureStatusResult = {
+    sessionId: string;
+    command: CaptureCommand;
+    captureStatus: CaptureSessionStatus;
+    overlayStatus?: OverlaySession["status"];
+    stage: VisualCaptureStage;
+    guidance: VisualCaptureGuidance;
+    clientHints: VisualCaptureClientHints;
+    captureSession: CaptureSession;
+    overlaySession?: OverlaySession;
+    result?: CaptureBundle;
+};
+export type AwaitVisualCaptureResult = {
+    sessionId: string;
+    stage: VisualCaptureStage;
+    guidance: VisualCaptureGuidance;
+    clientHints: VisualCaptureClientHints;
+    waitResult: AwaitCaptureSessionResult;
+    result?: CaptureBundle;
+};

package/dist/flow/types.js

@@ -0,0 +1 @@
+export {};

package/dist/flow/visual-capture-flow.d.ts

@@ -0,0 +1,13 @@
+import type { LocalOverlayAgent } from "../overlay/local-overlay-agent.js";
+import { CaptureSessionService } from "../session/capture-session-service.js";
+import type { CaptureCommand } from "../types/capture.js";
+import type { AwaitVisualCaptureResult, BeginVisualCaptureResult, VisualCaptureStatusResult } from "./types.js";
+export declare class VisualCaptureFlow {
+    private readonly captureSessions;
+    private readonly overlayAgent;
+    constructor(captureSessions: CaptureSessionService, overlayAgent: LocalOverlayAgent);
+    begin(command: CaptureCommand, ttlMs?: number): BeginVisualCaptureResult;
+    getStatus(sessionId: string): VisualCaptureStatusResult;
+    awaitResult(sessionId: string, timeoutMs?: number): Promise<AwaitVisualCaptureResult>;
+    private safeGetOverlaySession;
+}

package/dist/flow/visual-capture-flow.js

@@ -0,0 +1,196 @@
+const guidanceForStage = (stage, command, overlaySession) => {
+    switch (stage) {
+        case "armed":
+            return {
+                title: `${command} armed`,
+                message: "Capture mode is active. Switch to the target app, then start your selection.",
+                nextActions: ["switch_to_target", "select_region", "cancel"],
+                primaryAction: "switch_to_target",
+                footerHint: "The overlay stays armed until you select a region or cancel."
+            };
+        case "selecting":
+            return {
+                title: "Select the target area",
+                message: "Drag a region around the part of the screen you want the model to inspect.",
+                nextActions: ["select_region", "cancel"],
+                primaryAction: "select_region",
+                highlightedShortcut: overlaySession?.shortcuts.select,
+                footerHint: "A tighter crop gives better context and lower payload size."
+            };
+        case "annotating":
+            return {
+                title: command === "see" ? "Add guidance for the model" : "Optional markup",
+                message: command === "see"
+                    ? "Use boxes, arrows, text, or redaction to show the model exactly what matters."
+                    : "You can send now, or add quick annotations if a highlight would help.",
+                nextActions: ["annotate", "send", "cancel"],
+                primaryAction: command === "see" ? "annotate" : "send",
+                highlightedShortcut: overlaySession?.shortcuts[overlaySession?.activeTool ?? "select"],
+                footerHint: "Press the tool shortcut once to switch tools quickly."
+            };
+        case "ready_to_send":
+            return {
+                title: "Ready to send",
+                message: "The selection is ready. Send it back to the LLM when you are satisfied with the preview.",
+                nextActions: ["send", "cancel"],
+                primaryAction: "send",
+                footerHint: "If needed, you can still reposition the selection before sending."
+            };
+        case "completed":
+            return {
+                title: "Capture delivered",
+                message: "The visual bundle is complete and ready to attach to the current LLM interaction.",
+                nextActions: ["done"],
+                primaryAction: "done",
+                footerHint: "The image remained in memory only and was not persisted by default."
+            };
+        case "cancelled":
+            return {
+                title: "Capture cancelled",
+                message: "The visual capture flow was cancelled before send.",
+                nextActions: ["retry", "done"],
+                primaryAction: "retry"
+            };
+        case "failed":
+            return {
+                title: "Capture failed",
+                message: "The capture flow failed. Review the session error details and retry if needed.",
+                nextActions: ["retry", "done"],
+                primaryAction: "retry"
+            };
+        case "expired":
+            return {
+                title: "Capture expired",
+                message: "The capture session expired before completion. Start a new one to continue.",
+                nextActions: ["retry", "done"],
+                primaryAction: "retry"
+            };
+        case "timed_out":
+            return {
+                title: "Still waiting",
+                message: "The client-side wait timed out, but the capture session may still be active.",
+                nextActions: ["wait", "cancel", "done"],
+                primaryAction: "wait",
+                footerHint: "Use getVisualCaptureStatus to refresh the UI without losing progress."
+            };
+    }
+};
+const clientHintsForStage = (stage) => {
+    switch (stage) {
+        case "completed":
+            return { statusTone: "success", showToolbar: false, showPreview: true, allowInlineTips: false };
+        case "failed":
+            return { statusTone: "error", showToolbar: false, showPreview: false, allowInlineTips: true };
+        case "cancelled":
+        case "expired":
+            return { statusTone: "warning", showToolbar: false, showPreview: false, allowInlineTips: true };
+        case "timed_out":
+            return { statusTone: "warning", showToolbar: true, showPreview: true, allowInlineTips: true };
+        case "armed":
+        case "selecting":
+            return { statusTone: "info", showToolbar: true, showPreview: false, allowInlineTips: true };
+        case "annotating":
+        case "ready_to_send":
+            return { statusTone: "info", showToolbar: true, showPreview: true, allowInlineTips: true };
+    }
+};
+const deriveStage = (command, captureSession, overlaySession) => {
+    if (captureSession.status === "completed") {
+        return "completed";
+    }
+    if (captureSession.status === "cancelled") {
+        return "cancelled";
+    }
+    if (captureSession.status === "failed") {
+        return "failed";
+    }
+    if (captureSession.status === "expired") {
+        return "expired";
+    }
+    if (!overlaySession) {
+        return "armed";
+    }
+    if (overlaySession.status === "armed") {
+        return "selecting";
+    }
+    if (overlaySession.status === "selected") {
+        const hasAnnotations = overlaySession.annotationSummary.total > 0;
+        return hasAnnotations || command === "see" ? "annotating" : "ready_to_send";
+    }
+    if (overlaySession.status === "sent") {
+        return "completed";
+    }
+    if (overlaySession.status === "cancelled") {
+        return "cancelled";
+    }
+    if (overlaySession.status === "failed") {
+        return "failed";
+    }
+    if (overlaySession.status === "expired") {
+        return "expired";
+    }
+    return "armed";
+};
+export class VisualCaptureFlow {
+    captureSessions;
+    overlayAgent;
+    constructor(captureSessions, overlayAgent) {
+        this.captureSessions = captureSessions;
+        this.overlayAgent = overlayAgent;
+    }
+    begin(command, ttlMs) {
+        const captureSession = this.captureSessions.startSession({ command, ttlMs });
+        const overlaySession = this.overlayAgent.launch(captureSession.id);
+        const stage = deriveStage(command, captureSession, overlaySession);
+        return {
+            sessionId: captureSession.id,
+            command,
+            captureSession,
+            overlaySession,
+            stage,
+            guidance: guidanceForStage(stage, command, overlaySession),
+            clientHints: clientHintsForStage(stage)
+        };
+    }
+    getStatus(sessionId) {
+        const captureSession = this.captureSessions.getSession(sessionId);
+        const overlaySession = this.safeGetOverlaySession(sessionId);
+        const stage = deriveStage(captureSession.command, captureSession, overlaySession);
+        return {
+            sessionId,
+            command: captureSession.command,
+            captureStatus: captureSession.status,
+            overlayStatus: overlaySession?.status,
+            stage,
+            guidance: guidanceForStage(stage, captureSession.command, overlaySession),
+            clientHints: clientHintsForStage(stage),
+            captureSession,
+            overlaySession,
+            result: captureSession.result
+        };
+    }
+    async awaitResult(sessionId, timeoutMs) {
+        const waitResult = await this.captureSessions.awaitSession({ sessionId, timeoutMs });
+        const command = waitResult.session.command;
+        const overlaySession = this.safeGetOverlaySession(sessionId);
+        const stage = waitResult.outcome === "timed_out"
+            ? "timed_out"
+            : deriveStage(command, waitResult.session, overlaySession);
+        return {
+            sessionId,
+            stage,
+            guidance: guidanceForStage(stage, command, overlaySession),
+            clientHints: clientHintsForStage(stage),
+            waitResult,
+            result: waitResult.outcome === "completed" ? waitResult.result : undefined
+        };
+    }
+    safeGetOverlaySession(sessionId) {
+        try {
+            return this.overlayAgent.get(sessionId);
+        }
+        catch {
+            return undefined;
+        }
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/index.js

@@ -0,0 +1,3 @@
+import { VisualContextServer } from "./server.js";
+const server = new VisualContextServer();
+server.start();

package/dist/logging/logger.d.ts

@@ -0,0 +1,15 @@
+export interface Logger {
+    debug(message: string, context?: Record<string, unknown>): void;
+    info(message: string, context?: Record<string, unknown>): void;
+    warn(message: string, context?: Record<string, unknown>): void;
+    error(message: string, context?: Record<string, unknown>): void;
+}
+export declare class ConsoleLogger implements Logger {
+    private readonly scope;
+    constructor(scope: string);
+    debug(message: string, context?: Record<string, unknown>): void;
+    info(message: string, context?: Record<string, unknown>): void;
+    warn(message: string, context?: Record<string, unknown>): void;
+    error(message: string, context?: Record<string, unknown>): void;
+    private write;
+}

package/dist/logging/logger.js

@@ -0,0 +1,28 @@
+export class ConsoleLogger {
+    scope;
+    constructor(scope) {
+        this.scope = scope;
+    }
+    debug(message, context) {
+        this.write("debug", message, context);
+    }
+    info(message, context) {
+        this.write("info", message, context);
+    }
+    warn(message, context) {
+        this.write("warn", message, context);
+    }
+    error(message, context) {
+        this.write("error", message, context);
+    }
+    write(level, message, context) {
+        const payload = {
+            time: new Date().toISOString(),
+            level,
+            scope: this.scope,
+            message,
+            ...(context ? { context } : {})
+        };
+        process.stderr.write(`${JSON.stringify(payload)}\n`);
+    }
+}

package/dist/mcp/stdio-server.d.ts

@@ -0,0 +1,19 @@
+import type { Logger } from "../logging/logger.js";
+import type { VisualContextServer } from "../server.js";
+export declare class McpStdioServer {
+    private readonly app;
+    private readonly logger;
+    private readonly input;
+    private readonly output;
+    private buffer;
+    private initialized;
+    constructor(app: VisualContextServer, logger: Logger, input?: NodeJS.ReadStream & {
+        fd: 0;
+    }, output?: NodeJS.WriteStream & {
+        fd: 1;
+    });
+    start(): void;
+    private processBuffer;
+    private handleMessage;
+    private writeMessage;
+}