agent-vision-mcp 0.1.0

package/dist/capture/in-memory-image-compositor.d.ts

@@ -0,0 +1,5 @@
+import type { CaptureImage } from "../types/capture.js";
+import type { CaptureRenderManifest, RawScreenCapture } from "./types.js";
+export declare class InMemoryImageCompositor {
+    compose(source: RawScreenCapture, manifest: CaptureRenderManifest): CaptureImage;
+}

package/dist/capture/in-memory-image-compositor.js

@@ -0,0 +1,34 @@
+export class InMemoryImageCompositor {
+    compose(source, manifest) {
+        if (manifest.annotations.length === 0) {
+            const rawBytes = Buffer.from(source.bytesBase64, "base64");
+            return {
+                mimeType: source.mimeType,
+                bytesBase64: source.bytesBase64,
+                width: source.width,
+                height: source.height,
+                byteLength: rawBytes.byteLength,
+                sourceWidth: source.width,
+                sourceHeight: source.height,
+                backend: source.backend,
+                persisted: false
+            };
+        }
+        const renderedPayload = Buffer.from(JSON.stringify({
+            kind: "phase5-composited-capture",
+            source,
+            manifest
+        }));
+        return {
+            mimeType: "image/png",
+            bytesBase64: renderedPayload.toString("base64"),
+            width: manifest.crop.width,
+            height: manifest.crop.height,
+            byteLength: renderedPayload.byteLength,
+            sourceWidth: source.width,
+            sourceHeight: source.height,
+            backend: `${source.backend}+in-memory-compositor`,
+            persisted: false
+        };
+    }
+}

package/dist/capture/linux-portal-screenshot-provider.d.ts

@@ -0,0 +1,8 @@
+import type { Logger } from "../logging/logger.js";
+import type { ScreenCaptureProvider } from "./screen-capture-provider.js";
+import type { RawScreenCapture, ScreenCaptureProviderRequest } from "./types.js";
+export declare class LinuxPortalScreenshotProvider implements ScreenCaptureProvider {
+    private readonly logger;
+    constructor(logger: Logger);
+    capture(request: ScreenCaptureProviderRequest): Promise<RawScreenCapture>;
+}

package/dist/capture/linux-portal-screenshot-provider.js

@@ -0,0 +1,181 @@
+import { randomUUID } from "node:crypto";
+import { readFile, unlink } from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import dbus, { Variant } from "dbus-next";
+import { readPngDimensions } from "./png-metadata.js";
+const PORTAL_DESTINATION = "org.freedesktop.portal.Desktop";
+const PORTAL_OBJECT_PATH = "/org/freedesktop/portal/desktop";
+const SCREENSHOT_INTERFACE = "org.freedesktop.portal.Screenshot";
+const PROPERTIES_INTERFACE = "org.freedesktop.DBus.Properties";
+const REQUEST_INTERFACE = "org.freedesktop.portal.Request";
+const SCREENSHOT_TIMEOUT_MS = 20_000;
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const isVariantRecord = (value) => typeof value === "object" && value !== null;
+const readVariantNumber = (value) => {
+    if (typeof value === "number") {
+        return value;
+    }
+    if (typeof value === "bigint") {
+        return Number(value);
+    }
+    throw new Error("Expected numeric D-Bus variant value");
+};
+const readVariantString = (value) => {
+    if (typeof value !== "string" || value.trim() === "") {
+        throw new Error("Expected string D-Bus variant value");
+    }
+    return value;
+};
+const withTimeout = async (promise, timeoutMs, message) => {
+    let timeoutId;
+    try {
+        return await Promise.race([
+            promise,
+            new Promise((_, reject) => {
+                timeoutId = setTimeout(() => reject(new Error(message)), timeoutMs);
+            })
+        ]);
+    }
+    finally {
+        if (timeoutId) {
+            clearTimeout(timeoutId);
+        }
+    }
+};
+const buildRequestPath = (busName, token) => {
+    if (!busName.startsWith(":")) {
+        throw new Error(`Unexpected D-Bus unique name: ${busName}`);
+    }
+    return `/org/freedesktop/portal/desktop/request/${busName.slice(1).replaceAll('.', '_')}/${token}`;
+};
+const waitForBusName = async (bus) => {
+    for (let attempt = 0; attempt < 50; attempt += 1) {
+        if (typeof bus.name === "string" && bus.name.trim() !== "") {
+            return bus.name;
+        }
+        await delay(20);
+    }
+    throw new Error("Timed out waiting for a D-Bus unique name");
+};
+const waitForRequestResponse = async (bus, requestPath, timeoutMs) => {
+    const matchRule = `type='signal',sender='${PORTAL_DESTINATION}',interface='${REQUEST_INTERFACE}',member='Response',path='${requestPath}'`;
+    await bus._addMatch(matchRule);
+    let onMessage;
+    try {
+        return await withTimeout(new Promise((resolve, reject) => {
+            onMessage = (message) => {
+                if (message.path !== requestPath ||
+                    message.interface !== REQUEST_INTERFACE ||
+                    message.member !== "Response") {
+                    return;
+                }
+                if (onMessage) {
+                    bus.off("message", onMessage);
+                }
+                try {
+                    const [responseCodeRaw, resultsRaw] = message.body;
+                    if (!isVariantRecord(resultsRaw)) {
+                        reject(new Error("Portal screenshot response did not include a result map"));
+                        return;
+                    }
+                    resolve({
+                        responseCode: readVariantNumber(responseCodeRaw),
+                        results: resultsRaw
+                    });
+                }
+                catch (error) {
+                    reject(error);
+                }
+            };
+            bus.on("message", onMessage);
+        }), timeoutMs, `Timed out waiting for screenshot portal response after ${timeoutMs}ms`);
+    }
+    finally {
+        if (onMessage) {
+            bus.off("message", onMessage);
+        }
+        try {
+            await bus._removeMatch(matchRule);
+        }
+        catch {
+            // Ignore cleanup failures during teardown.
+        }
+    }
+};
+const readUriFromResults = (results) => {
+    const uri = results.uri;
+    if (!uri) {
+        throw new Error("Portal screenshot response did not include a uri result");
+    }
+    return readVariantString(uri.value);
+};
+const deleteFileIfPossible = async (uri) => {
+    if (!uri.startsWith("file://")) {
+        return;
+    }
+    try {
+        await unlink(fileURLToPath(uri));
+    }
+    catch {
+        // Best-effort cleanup only.
+    }
+};
+export class LinuxPortalScreenshotProvider {
+    logger;
+    constructor(logger) {
+        this.logger = logger;
+    }
+    async capture(request) {
+        if (process.platform !== "linux") {
+            throw new Error("Linux portal screenshot provider can only run on Linux");
+        }
+        const bus = dbus.sessionBus();
+        try {
+            const busName = await waitForBusName(bus);
+            const handleToken = `llmvision_${randomUUID().replaceAll("-", "")}`;
+            const requestPath = buildRequestPath(busName, handleToken);
+            const responsePromise = waitForRequestResponse(bus, requestPath, SCREENSHOT_TIMEOUT_MS);
+            const portalObject = await bus.getProxyObject(PORTAL_DESTINATION, PORTAL_OBJECT_PATH);
+            const properties = portalObject.getInterface(PROPERTIES_INTERFACE);
+            const screenshotVersion = await properties.Get(SCREENSHOT_INTERFACE, "version");
+            this.logger.debug("Detected portal screenshot interface", {
+                sessionId: request.sessionId,
+                version: readVariantNumber(screenshotVersion.value)
+            });
+            const screenshot = portalObject.getInterface(SCREENSHOT_INTERFACE);
+            const returnedHandle = await screenshot.Screenshot("", {
+                handle_token: new Variant("s", handleToken),
+                interactive: new Variant("b", false),
+                modal: new Variant("b", false)
+            });
+            this.logger.debug("Issued portal screenshot request", {
+                sessionId: request.sessionId,
+                requestPath,
+                returnedHandle,
+                activeAppName: request.activeAppName,
+                activeWindowTitle: request.activeWindowTitle
+            });
+            const { responseCode, results } = await responsePromise;
+            if (responseCode !== 0) {
+                throw new Error(`Screenshot portal request ended with response code ${responseCode}`);
+            }
+            const uri = readUriFromResults(results);
+            const filePath = fileURLToPath(uri);
+            const imageBytes = await readFile(filePath);
+            const dimensions = readPngDimensions(imageBytes);
+            await deleteFileIfPossible(uri);
+            return {
+                mimeType: "image/png",
+                bytesBase64: imageBytes.toString("base64"),
+                width: dimensions.width,
+                height: dimensions.height,
+                capturedAt: new Date().toISOString(),
+                displayId: request.displayId,
+                backend: "xdg-portal-screenshot"
+            };
+        }
+        finally {
+            bus.disconnect();
+        }
+    }
+}

package/dist/capture/mock-screen-capture-provider.d.ts

@@ -0,0 +1,5 @@
+import type { ScreenCaptureProvider } from "./screen-capture-provider.js";
+import type { RawScreenCapture, ScreenCaptureProviderRequest } from "./types.js";
+export declare class MockScreenCaptureProvider implements ScreenCaptureProvider {
+    capture(request: ScreenCaptureProviderRequest): Promise<RawScreenCapture>;
+}

package/dist/capture/mock-screen-capture-provider.js

@@ -0,0 +1,22 @@
+export class MockScreenCaptureProvider {
+    async capture(request) {
+        const descriptor = {
+            kind: "phase5-mock-screen-capture",
+            sessionId: request.sessionId,
+            command: request.command,
+            displayId: request.displayId ?? "display-1",
+            activeAppName: request.activeAppName ?? "Prototype App",
+            activeWindowTitle: request.activeWindowTitle ?? "Prototype Window",
+            generatedAt: new Date().toISOString()
+        };
+        return {
+            mimeType: "image/png",
+            bytesBase64: Buffer.from(JSON.stringify(descriptor)).toString("base64"),
+            width: 1920,
+            height: 1080,
+            capturedAt: new Date().toISOString(),
+            displayId: request.displayId ?? "display-1",
+            backend: "mock-screen-provider"
+        };
+    }
+}

package/dist/capture/png-metadata.d.ts

@@ -0,0 +1,5 @@
+export type PngDimensions = {
+    width: number;
+    height: number;
+};
+export declare const readPngDimensions: (buffer: Buffer) => PngDimensions;

package/dist/capture/png-metadata.js

@@ -0,0 +1,18 @@
+const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
+export const readPngDimensions = (buffer) => {
+    if (buffer.byteLength < 24) {
+        throw new Error("PNG buffer is too small to contain dimensions");
+    }
+    const signature = buffer.subarray(0, PNG_SIGNATURE.byteLength);
+    if (!signature.equals(PNG_SIGNATURE)) {
+        throw new Error("Expected PNG signature in screenshot buffer");
+    }
+    const chunkType = buffer.subarray(12, 16).toString("ascii");
+    if (chunkType !== "IHDR") {
+        throw new Error("Expected IHDR chunk at the start of the PNG payload");
+    }
+    return {
+        width: buffer.readUInt32BE(16),
+        height: buffer.readUInt32BE(20)
+    };
+};

package/dist/capture/screen-capture-provider.d.ts

@@ -0,0 +1,4 @@
+import type { RawScreenCapture, ScreenCaptureProviderRequest } from "./types.js";
+export interface ScreenCaptureProvider {
+    capture(request: ScreenCaptureProviderRequest): Promise<RawScreenCapture>;
+}

package/dist/capture/screen-capture-provider.js

@@ -0,0 +1 @@
+export {};

package/dist/capture/types.d.ts

@@ -0,0 +1,38 @@
+import type { Annotation } from "../types/annotation.js";
+import type { CaptureCommand, CaptureContext, SelectionBounds } from "../types/capture.js";
+export type RawScreenCapture = {
+    mimeType: "image/png";
+    bytesBase64: string;
+    width: number;
+    height: number;
+    capturedAt: string;
+    displayId?: string;
+    backend: string;
+};
+export type ScreenCaptureProviderRequest = {
+    sessionId: string;
+    command: CaptureCommand;
+    displayId?: string;
+    activeAppName?: string;
+    activeWindowTitle?: string;
+};
+export type CapturePipelineInput = {
+    sessionId: string;
+    command: CaptureCommand;
+    selection: SelectionBounds;
+    annotations: Annotation[];
+    context?: Omit<CaptureContext, "capturedAt">;
+};
+export type CaptureRenderManifest = {
+    sessionId: string;
+    command: CaptureCommand;
+    sourceImage: {
+        width: number;
+        height: number;
+        displayId?: string;
+        backend: string;
+    };
+    crop: SelectionBounds;
+    annotations: Annotation[];
+    context?: Omit<CaptureContext, "capturedAt">;
+};

package/dist/capture/types.js

@@ -0,0 +1 @@
+export {};

package/dist/cdp-demo.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/cdp-demo.js

@@ -0,0 +1,41 @@
+import { VisualContextServer } from "./server.js";
+const main = async () => {
+    const server = new VisualContextServer();
+    server.start();
+    const status = await server.callTool("getBrowserCdpStatus");
+    const rawTabs = await server.callTool("discoverBrowserTabsViaCdp");
+    const liveTabs = await server.callTool("refreshLiveBrowserTabs");
+    const cachedLiveTabs = await server.callTool("listLiveBrowserTabs");
+    const pruned = await server.callTool("pruneStaleLiveBrowserTabs", {
+        maxAgeMs: 5 * 60 * 1000
+    });
+    const resolvedActive = await server.callTool("resolveLiveBrowserTab");
+    const resolvedQuery = await server.callTool("resolveLiveBrowserTab", {
+        query: "docs"
+    });
+    const screenshot = await server.callTool("captureResolvedBrowserTabScreenshot", {
+        query: "docs"
+    });
+    const context = await server.callTool("getResolvedBrowserTabContext", {
+        query: "docs"
+    });
+    const seeActive = await server.callTool("seeBrowserTabViaCdp");
+    const seeQuery = await server.callTool("seeBrowserTabViaCdp", {
+        query: "docs"
+    });
+    console.log("cdp-status", JSON.stringify(status, null, 2));
+    console.log("cdp-tabs", JSON.stringify(rawTabs, null, 2));
+    console.log("live-browser-tabs", JSON.stringify(liveTabs, null, 2));
+    console.log("cached-live-browser-tabs", JSON.stringify(cachedLiveTabs, null, 2));
+    console.log("pruned-stale-tabs", JSON.stringify(pruned, null, 2));
+    console.log("resolved-active", JSON.stringify(resolvedActive, null, 2));
+    console.log("resolved-query", JSON.stringify(resolvedQuery, null, 2));
+    console.log("captured-screenshot", JSON.stringify(screenshot, null, 2));
+    console.log("resolved-context", JSON.stringify(context, null, 2));
+    console.log("see-active", JSON.stringify(seeActive, null, 2));
+    console.log("see-query", JSON.stringify(seeQuery, null, 2));
+};
+void main().catch((error) => {
+    console.error(error);
+    process.exitCode = 1;
+});

package/dist/demo.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/demo.js

@@ -0,0 +1,54 @@
+import { VisualContextServer } from "./server.js";
+const ensureBeginResult = (value) => {
+    if (!value || typeof value !== "object" || !("sessionId" in value) || typeof value.sessionId !== "string") {
+        throw new Error("Unexpected beginVisualCapture response");
+    }
+    return value;
+};
+const run = async () => {
+    const server = new VisualContextServer();
+    server.start();
+    const started = ensureBeginResult(await server.callTool("beginVisualCapture", { command: "see", ttlMs: 5_000 }));
+    console.log("begin", JSON.stringify(started, null, 2));
+    await server.callTool("selectOverlayRegion", {
+        sessionId: started.sessionId,
+        x: 120,
+        y: 96,
+        width: 640,
+        height: 360,
+        activeAppName: "Prototype Browser",
+        activeWindowTitle: "Phase 8 Demo",
+        displayId: "display-1"
+    });
+    await server.callTool("setOverlayActiveTool", {
+        sessionId: started.sessionId,
+        tool: "rect"
+    });
+    await server.callTool("addOverlayAnnotation", {
+        sessionId: started.sessionId,
+        annotation: {
+            type: "rect",
+            x: 140,
+            y: 120,
+            width: 300,
+            height: 120,
+            label: "Problem area"
+        }
+    });
+    const statusBeforeSend = await server.callTool("getVisualCaptureStatus", {
+        sessionId: started.sessionId
+    });
+    console.log("status-before-send", JSON.stringify(statusBeforeSend, null, 2));
+    await server.callTool("sendOverlayCaptureSession", {
+        sessionId: started.sessionId
+    });
+    const finalResult = await server.callTool("awaitVisualCaptureResult", {
+        sessionId: started.sessionId,
+        timeoutMs: 5_000
+    });
+    console.log("awaited", JSON.stringify(finalResult, null, 2));
+};
+run().catch((error) => {
+    console.error(error);
+    process.exitCode = 1;
+});

package/dist/desktop/capture-now.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/desktop/capture-now.js

@@ -0,0 +1,48 @@
+import { app, screen } from "electron";
+import { OverlayDesktopController } from "./controller.js";
+const controller = new OverlayDesktopController();
+const getDefaultDisplaySelection = () => {
+    const display = screen.getPrimaryDisplay();
+    const { x, y, width, height } = display.workArea;
+    return {
+        x,
+        y,
+        width,
+        height,
+        displayId: String(display.id),
+        activeAppName: "Current Display",
+        activeWindowTitle: "Wayland portal capture"
+    };
+};
+const captureActiveWindow = async (payload) => {
+    const capturePayload = {
+        command: payload?.command ?? "see",
+        selection: payload?.selection ?? getDefaultDisplaySelection()
+    };
+    console.log("[capture-now] capture requested", {
+        command: capturePayload.command,
+        selection: capturePayload.selection
+    });
+    const result = await controller.captureActiveWindow(capturePayload);
+    console.log("[capture-now] capture completed", {
+        stage: result.stage,
+        outcome: result.waitResult.outcome,
+        sessionId: result.sessionId,
+        backend: result.result?.image.backend,
+        width: result.result?.image.width,
+        height: result.result?.image.height,
+        byteLength: result.result?.image.byteLength
+    });
+};
+app.whenReady().then(async () => {
+    try {
+        await captureActiveWindow();
+        app.quit();
+    }
+    catch (error) {
+        console.error("[capture-now] capture failed", {
+            error: String(error?.message ?? error)
+        });
+        app.exit(1);
+    }
+});

package/dist/desktop/controller.d.ts

@@ -0,0 +1,25 @@
+import type { AwaitVisualCaptureResult, BeginVisualCaptureResult, VisualCaptureStatusResult } from "../flow/types.js";
+import type { Annotation } from "../types/annotation.js";
+import type { OverlayTool } from "../overlay/types.js";
+import type { SelectionBounds } from "../types/capture.js";
+import type { ActiveWindowCapturePayload } from "./types.js";
+export declare class OverlayDesktopController {
+    private readonly server;
+    private currentSessionId?;
+    constructor();
+    begin(command?: "see" | "clip", ttlMs?: number): Promise<BeginVisualCaptureResult>;
+    getStatus(sessionId?: string): Promise<VisualCaptureStatusResult>;
+    selectRegion(bounds: SelectionBounds & {
+        displayId?: string;
+        activeAppName?: string;
+        activeWindowTitle?: string;
+    }, sessionId?: string): Promise<VisualCaptureStatusResult>;
+    setTool(tool: OverlayTool, sessionId?: string): Promise<VisualCaptureStatusResult>;
+    addAnnotation(annotation: Annotation, sessionId?: string): Promise<VisualCaptureStatusResult>;
+    clearAnnotations(sessionId?: string): Promise<VisualCaptureStatusResult>;
+    send(sessionId?: string): Promise<AwaitVisualCaptureResult>;
+    cancel(sessionId?: string): Promise<VisualCaptureStatusResult>;
+    captureActiveWindow(payload: ActiveWindowCapturePayload): Promise<AwaitVisualCaptureResult>;
+    getCurrentSessionId(): string | undefined;
+    private requireSessionId;
+}

package/dist/desktop/controller.js

@@ -0,0 +1,77 @@
+import { VisualContextServer } from "../server.js";
+export class OverlayDesktopController {
+    server = new VisualContextServer();
+    currentSessionId;
+    constructor() {
+        this.server.start();
+    }
+    async begin(command = "see", ttlMs = 15 * 60 * 1000) {
+        const started = (await this.server.callTool("beginVisualCapture", {
+            command,
+            ttlMs
+        }));
+        this.currentSessionId = started.sessionId;
+        return started;
+    }
+    async getStatus(sessionId = this.requireSessionId()) {
+        return (await this.server.callTool("getVisualCaptureStatus", {
+            sessionId
+        }));
+    }
+    async selectRegion(bounds, sessionId = this.requireSessionId()) {
+        await this.server.callTool("selectOverlayRegion", {
+            sessionId,
+            ...bounds
+        });
+        return this.getStatus(sessionId);
+    }
+    async setTool(tool, sessionId = this.requireSessionId()) {
+        await this.server.callTool("setOverlayActiveTool", {
+            sessionId,
+            tool
+        });
+        return this.getStatus(sessionId);
+    }
+    async addAnnotation(annotation, sessionId = this.requireSessionId()) {
+        await this.server.callTool("addOverlayAnnotation", {
+            sessionId,
+            annotation
+        });
+        return this.getStatus(sessionId);
+    }
+    async clearAnnotations(sessionId = this.requireSessionId()) {
+        await this.server.callTool("clearOverlayAnnotations", {
+            sessionId
+        });
+        return this.getStatus(sessionId);
+    }
+    async send(sessionId = this.requireSessionId()) {
+        await this.server.callTool("sendOverlayCaptureSession", {
+            sessionId
+        });
+        return (await this.server.callTool("awaitVisualCaptureResult", {
+            sessionId,
+            timeoutMs: 5_000
+        }));
+    }
+    async cancel(sessionId = this.requireSessionId()) {
+        await this.server.callTool("cancelOverlayCaptureSession", {
+            sessionId
+        });
+        return this.getStatus(sessionId);
+    }
+    async captureActiveWindow(payload) {
+        const started = await this.begin(payload.command ?? "see");
+        await this.selectRegion(payload.selection, started.sessionId);
+        return this.send(started.sessionId);
+    }
+    getCurrentSessionId() {
+        return this.currentSessionId;
+    }
+    requireSessionId() {
+        if (!this.currentSessionId) {
+            throw new Error("No active overlay session");
+        }
+        return this.currentSessionId;
+    }
+}

package/dist/desktop/main.d.ts

@@ -0,0 +1 @@
+export {};