@aexol/spectral 0.9.211 → 0.9.215

package/dist/extensions/desktop-screenshot/index.js

@@ -18,13 +18,17 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { randomUUID } from "node:crypto";
+import { uploadImage } from "../../backend/generated-images.js";
+import { resolveBackendRouting } from "../../backend/machine-routing.js";
+import { resizeImage } from "../../sdk/coding-agent/utils/image-resize.js";
+import { formatImageBytes, MAX_INLINE_IMAGE_BASE64_CHARS, } from "../../server/image-attachments.js";
 const SCREENSHOTS_DIR = ".screenshots";
-function tmpDir() {
+export function tmpDir() {
     const d = join(tmpdir(), "spectral-screenshots");
     mkdirSync(d, { recursive: true });
     return d;
 }
-function saveScreenshot(buf, format, cwd) {
+export function saveScreenshot(buf, format, cwd) {
     const dir = join(cwd, SCREENSHOTS_DIR);
     mkdirSync(dir, { recursive: true });
     const now = new Date();
@@ -176,6 +180,221 @@ function darwinWindowScreenshot(windowId, options) {
         });
     });
 }
+// ===========================================================================
+// macOS coordinate-space normalization (Retina pixels → logical points)
+// ===========================================================================
+/**
+ * `screencapture` returns images in *physical* pixels — 2× (or 3×) the
+ * logical resolution on Retina displays — while the `desktop_*` tools
+ * (CGEvent) operate in *logical* screen points. A model reading coordinates
+ * straight off an unnormalized capture would click ~2× too far. The helpers
+ * below probe the display geometry (JXA/AppKit, logical points), resample
+ * the capture to a 1:1 logical-points image with `sips`, and produce
+ * metadata that states the applied scale explicitly.
+ */
+const COORD_SPACE_NOTE = "Coordinates for desktop_* tools are logical screen points — the image above is already scaled 1:1 to that space.";
+const DARWIN_GEOMETRY_TTL_MS = 10_000;
+let darwinGeometryCache = null;
+/** Test hook: forget the cached geometry so the next probe hits osascript again. */
+export function resetDarwinScreenGeometryCache() {
+    darwinGeometryCache = null;
+}
+function runCommand(cmd, args, timeoutMs) {
+    return new Promise((resolve, reject) => {
+        execFile(cmd, args, { timeout: timeoutMs }, (err, stdout) => {
+            if (err)
+                return reject(err);
+            resolve(stdout);
+        });
+    });
+}
+// NSScreen reports logical points. AppKit's global space is bottom-left
+// origin while CGEvent's is top-left origin of the primary display, so y is
+// flipped before the geometry is reported (the desktop_* tools are CGEvent).
+const DARWIN_GEOMETRY_JXA = [
+    "ObjC.import('AppKit');",
+    "const screens = $.NSScreen.screens;",
+    "const primaryF = screens.objectAtIndex(0).frame;",
+    "const mainF = $.NSScreen.mainScreen.frame;",
+    "JSON.stringify({",
+    "	count: Number(screens.count),",
+    "	primary: { x: Number(primaryF.origin.x), y: Number(primaryF.origin.y), w: Number(primaryF.size.width), h: Number(primaryF.size.height) },",
+    "	main: { x: Number(mainF.origin.x), y: Number(mainF.origin.y), w: Number(mainF.size.width), h: Number(mainF.size.height) },",
+    "	backingScale: Number($.NSScreen.mainScreen.backingScaleFactor)",
+    "});",
+].join("\n");
+/**
+ * Logical display geometry via JXA/AppKit, cached for a few seconds (display
+ * arrangements rarely change mid-session). Resolves to `null` when the probe
+ * fails — captures are then left untouched and the tool result says the
+ * scale is unverified. Exported for tests.
+ */
+export async function darwinScreenGeometry() {
+    const now = Date.now();
+    if (darwinGeometryCache && now - darwinGeometryCache.at < DARWIN_GEOMETRY_TTL_MS) {
+        return darwinGeometryCache.geometry;
+    }
+    const geometry = await (async () => {
+        try {
+            const out = await runCommand("osascript", ["-l", "JavaScript", "-e", DARWIN_GEOMETRY_JXA], 5000);
+            const raw = JSON.parse(out.trim());
+            if (!raw || raw.count < 1 || !raw.primary || !raw.main || !Number.isFinite(raw.backingScale))
+                return null;
+            const toCgY = (r) => raw.primary.h - (r.y + r.h);
+            return {
+                displayCount: raw.count,
+                primaryOrigin: { x: raw.primary.x, y: toCgY(raw.primary) },
+                main: { x: raw.main.x, y: toCgY(raw.main), width: raw.main.w, height: raw.main.h },
+                backingScaleFactor: raw.backingScale,
+            };
+        }
+        catch {
+            return null;
+        }
+    })();
+    darwinGeometryCache = { at: now, geometry };
+    return geometry;
+}
+async function probeDarwinImageSize(path) {
+    try {
+        const out = await runCommand("sips", ["-g", "pixelWidth", "-g", "pixelHeight", path], 5000);
+        const w = /pixelWidth:\s*(\d+)/.exec(out);
+        const h = /pixelHeight:\s*(\d+)/.exec(out);
+        if (!w || !h)
+            return null;
+        return { width: Number(w[1]), height: Number(h[1]) };
+    }
+    catch {
+        return null;
+    }
+}
+function nearSize(a, b) {
+    return Math.abs(a - b) <= Math.max(2, b * 0.02);
+}
+function fmtScale(scale) {
+    return `${Math.round(scale * 100) / 100}×`;
+}
+/**
+ * Normalize a macOS capture so image pixels map 1:1 onto the logical screen
+ * points the desktop_* tools accept. Full-screen captures are resampled to
+ * the main screen's logical width when the file is a Retina (backing-scale)
+ * capture; window captures are resampled by the display's backing scale
+ * factor (their coordinates are window-local, which the metadata states).
+ * Never throws: on any probe failure the original buffer comes back with a
+ * caution note instead of a silently wrong scale. Exported for tests.
+ */
+export async function normalizeDarwinScreenshot(buf, format, mode) {
+    const unchanged = (metadata, details = {}) => ({ buffer: buf, metadata, details });
+    try {
+        const geometry = await darwinScreenGeometry();
+        const inPath = join(tmpDir(), `${randomUUID()}.${format}`);
+        writeFileSync(inPath, buf);
+        const size = await probeDarwinImageSize(inPath);
+        if (!size) {
+            const factor = geometry && geometry.backingScaleFactor > 1 ? Math.round(geometry.backingScaleFactor) : 2;
+            return unchanged(geometry
+                ? `Note: image size could not be verified (main screen: ${geometry.main.width}×${geometry.main.height} pt at ` +
+                    `${fmtScale(geometry.backingScaleFactor)} backing scale) — divide read-off coordinates by ${factor} on a ` +
+                    "Retina Mac before using desktop_* tools."
+                : "Note: image size and screen geometry could not be verified — divide read-off coordinates by 2 on a " +
+                    "Retina Mac before using desktop_* tools.");
+        }
+        let targetWidth = null;
+        let scaleApplied;
+        if (mode === "screen" && geometry) {
+            const logicalW = geometry.main.width;
+            if (nearSize(size.width, logicalW * geometry.backingScaleFactor) && logicalW < size.width) {
+                targetWidth = Math.round(logicalW);
+                scaleApplied = size.width / logicalW;
+            }
+            else if (nearSize(size.width, logicalW)) {
+                scaleApplied = 1;
+            }
+        }
+        else if (mode === "window" && geometry && geometry.backingScaleFactor > 1.01) {
+            targetWidth = Math.round(size.width / geometry.backingScaleFactor);
+            scaleApplied = geometry.backingScaleFactor;
+        }
+        let outBuf = buf;
+        let outWidth = size.width;
+        let outHeight = size.height;
+        let resampled = false;
+        if (targetWidth !== null && targetWidth > 0 && targetWidth < size.width) {
+            const outPath = join(tmpDir(), `${randomUUID()}.${format}`);
+            await runCommand("sips", ["--resampleWidth", String(targetWidth), inPath, "--out", outPath], 15000);
+            outBuf = readFileSync(outPath);
+            outWidth = targetWidth;
+            outHeight = Math.round(size.height * (targetWidth / size.width));
+            resampled = true;
+        }
+        const displayNote = geometry
+            ? `${geometry.displayCount} display(s); primary display global origin (${geometry.primaryOrigin.x}, ${geometry.primaryOrigin.y}) pt in the global CGEvent space` +
+                (geometry.main.x !== 0 || geometry.main.y !== 0
+                    ? `; main screen top-left at (${geometry.main.x}, ${geometry.main.y}) pt`
+                    : "")
+            : "";
+        const windowLocalNote = "Coordinates for desktop_* tools are global logical screen points — these are window-local, " +
+            "so offset them by the window's position (or capture the full desktop) before clicking.";
+        let metadata;
+        if (mode === "screen") {
+            if (resampled && geometry) {
+                metadata =
+                    `Image ${size.width}×${size.height} px is a ${fmtScale(scaleApplied ?? 2)} Retina capture → resampled to ` +
+                        `${outWidth}×${outHeight} px for the ${geometry.main.width}×${geometry.main.height} pt logical display. ` +
+                        `${COORD_SPACE_NOTE} ${displayNote}.`;
+            }
+            else if (scaleApplied === 1 && geometry) {
+                metadata =
+                    `Image ${outWidth}×${outHeight} px on the ${geometry.main.width}×${geometry.main.height} pt logical display ` +
+                        `(scale 1× — no resampling needed). ${COORD_SPACE_NOTE} ${displayNote}.`;
+            }
+            else if (geometry) {
+                metadata =
+                    `Image ${outWidth}×${outHeight} px does not match the main screen's geometry ` +
+                        `(${geometry.main.width}×${geometry.main.height} pt at ${fmtScale(geometry.backingScaleFactor)}; a secondary display?) — ` +
+                        "desktop_* tools take logical screen points, so verify the scale before converting read-off coordinates.";
+            }
+            else {
+                metadata =
+                    `Image ${outWidth}×${outHeight} px; screen geometry could not be probed — desktop_* tools take logical ` +
+                        "screen points, so on a Retina Mac (2×) divide read-off coordinates by 2 first.";
+            }
+        }
+        else {
+            metadata = resampled && geometry
+                ? `Window image ${size.width}×${size.height} px at ${fmtScale(scaleApplied ?? geometry.backingScaleFactor)} backing scale → resampled to ` +
+                    `${outWidth}×${outHeight} px (window-local logical points, assuming the window sits on the main display). ` +
+                    windowLocalNote
+                : geometry
+                    ? `Window image ${outWidth}×${outHeight} px at ${fmtScale(geometry.backingScaleFactor)} backing scale ` +
+                        `(window-local logical points, no resampling needed). ${windowLocalNote}`
+                    : `Window image ${outWidth}×${outHeight} px; screen geometry could not be probed — on a Retina Mac (2×) ` +
+                        "window captures are 2× physical pixels, so divide read-off coordinates by 2 before clicking.";
+        }
+        return {
+            buffer: outBuf,
+            metadata,
+            details: {
+                imageWidthPx: outWidth,
+                imageHeightPx: outHeight,
+                ...(geometry
+                    ? {
+                        logicalWidthPt: geometry.main.width,
+                        logicalHeightPt: geometry.main.height,
+                        displayCount: geometry.displayCount,
+                        primaryOriginPt: geometry.primaryOrigin,
+                    }
+                    : {}),
+                ...(scaleApplied !== undefined ? { scaleApplied } : {}),
+                resampled,
+            },
+        };
+    }
+    catch {
+        return unchanged("Note: coordinate-space normalization failed — desktop_* tools take logical screen points; " +
+            "on a Retina Mac (2×) divide read-off coordinates by 2 before clicking.");
+    }
+}
 function parseLinuxDisplaysOutput(output) {
     return output
         .split("\n")
@@ -272,9 +491,9 @@ function linuxWindowScreenshot(windowId, options) {
  * System.Windows.Forms / CopyFromScreen is initialized even in sessions that
  * default to MTA (avoids COM failures).
  */
-function runPowerShell(psPath) {
+export function runPowerShell(psPath, opts) {
     return new Promise((resolve, reject) => {
-        const child = crossSpawn("powershell.exe", ["-NoProfile", "-Sta", "-ExecutionPolicy", "Bypass", "-File", psPath], { timeout: 15000, windowsHide: true });
+        const child = crossSpawn("powershell.exe", ["-NoProfile", "-Sta", "-ExecutionPolicy", "Bypass", "-File", psPath], { timeout: opts?.timeoutMs ?? 15000, windowsHide: true });
         let stdout = "";
         let stderr = "";
         child.stdout?.on("data", (chunk) => { stdout += chunk.toString(); });
@@ -412,7 +631,7 @@ function win32WindowScreenshot(hwnd, options) {
 // ===========================================================================
 // Unified API
 // ===========================================================================
-function screenshot(options = {}) {
+export function screenshot(options = {}) {
     switch (process.platform) {
         case "darwin": return darwinFullScreenshot(options);
         case "linux": return linuxFullScreenshot(options);
@@ -420,7 +639,7 @@ function screenshot(options = {}) {
         default: return Promise.reject(new Error(`Unsupported platform: ${process.platform}`));
     }
 }
-function screenshotWindow(windowId, options, platform) {
+export function screenshotWindow(windowId, options, platform) {
     switch (platform) {
         case "darwin": return darwinWindowScreenshot(windowId, options);
         case "linux": return linuxWindowScreenshot(windowId, options);
@@ -428,7 +647,7 @@ function screenshotWindow(windowId, options, platform) {
         default: return Promise.reject(new Error(`Window screenshot not supported on ${platform}`));
     }
 }
-function listWindows() {
+export function listWindows() {
     switch (process.platform) {
         case "darwin": return listDarwinWindows();
         case "linux": return listLinuxWindows();
@@ -439,7 +658,7 @@ function listWindows() {
 // ===========================================================================
 // Helpers
 // ===========================================================================
-function formatWindowList(windows, maxRows = 40) {
+export function formatWindowList(windows, maxRows = 40) {
     if (windows.length === 0)
         return "No visible windows found.";
     const rows = windows.slice(0, maxRows);
@@ -454,6 +673,41 @@ function formatWindowList(windows, maxRows = 40) {
     return `${header}\n${body}`;
 }
 // ===========================================================================
+// Inline fallback sizing
+// ===========================================================================
+/**
+ * Last-resort inline carrier for a screenshot that could not be hosted.
+ *
+ * Returns the image unchanged when its base64 payload already fits
+ * `MAX_INLINE_IMAGE_BASE64_CHARS`, a downscaled/re-encoded copy when resizing
+ * gets it under the cap, and `null` when even that fails (no image resizer
+ * available, or the capture is absurdly large). Callers MUST treat `null` as
+ * "do not attach base64" and report the failure — never as "attach it anyway".
+ *
+ * Exported for tests.
+ */
+export async function fitInlineScreenshot(image, maxBase64Chars = MAX_INLINE_IMAGE_BASE64_CHARS) {
+    const data = image.data;
+    if (!data)
+        return null;
+    if (data.length <= maxBase64Chars)
+        return image;
+    let resized = null;
+    try {
+        resized = await resizeImage(image, { maxBytes: maxBase64Chars });
+    }
+    catch {
+        resized = null;
+    }
+    if (!resized || resized.data.length > maxBase64Chars)
+        return null;
+    return {
+        type: "image",
+        data: resized.data,
+        mimeType: resized.mimeType || image.mimeType,
+    };
+}
+// ===========================================================================
 // Extension registration
 // ===========================================================================
 export default async function desktopScreenshotExtension(ext) {
@@ -477,15 +731,101 @@ export default async function desktopScreenshotExtension(ext) {
             const p = params;
             const format = p.format ?? "png";
             const mimeType = (format === "jpg" || format === "jpeg") ? "image/jpeg" : "image/png";
-            const makeImageResult = (buf, label) => {
-                const base64 = buf.toString("base64");
+            /**
+             * A screenshot is worth megabytes of base64. When the CLI can reach
+             * the backend we upload it and put only the hosted url into the
+             * context; otherwise we fall back to inline base64, downscaled to
+             * the relay cap (best effort — a storage outage must never fail
+             * the screenshot, but it must also never dump megabytes of base64
+             * into the session history).
+             */
+            const makeImageResult = async (buf, label, coordMeta) => {
                 const savedPath = saveScreenshot(buf, format, ctx.cwd);
+                const routing = await resolveBackendRouting();
+                let imageBlock = {
+                    type: "image",
+                    data: buf.toString("base64"),
+                    mimeType,
+                };
+                let hostedUrl;
+                if (routing) {
+                    try {
+                        const upload = await uploadImage({
+                            backendUrl: routing.backendUrl,
+                            token: routing.machineJwt,
+                            // Copy: `buf` may be a pooled Buffer view.
+                            bytes: new Uint8Array(buf),
+                            contentType: mimeType,
+                            kind: "attachment",
+                            signal: _signal,
+                        });
+                        imageBlock = {
+                            type: "image",
+                            url: upload.url,
+                            imageId: upload.imageId,
+                            mimeType,
+                        };
+                        hostedUrl = upload.url;
+                    }
+                    catch (error) {
+                        process.stderr.write(`[desktop_screenshot] could not host the screenshot (${error instanceof Error ? error.message : String(error)}); ` +
+                            "falling back to a downscaled inline image.\n");
+                    }
+                }
+                // Inline fallback (no backend routing, or the upload failed):
+                // a full-screen retina PNG is worth several megabytes of
+                // base64, which would be dumped into the tool result AND
+                // persisted in the session JSONL. Downscale it to fit the
+                // relay cap, or refuse it with a clear message instead of
+                // silently shipping megabytes of base64.
+                if (!hostedUrl) {
+                    const fitted = await fitInlineScreenshot(imageBlock);
+                    if (!fitted) {
+                        const base64Chars = imageBlock.data?.length ?? 0;
+                        return {
+                            content: [
+                                {
+                                    type: "text",
+                                    text: `❌ ${label} was saved to ${savedPath} but NOT attached: the image is too large to send ` +
+                                        `inline (${formatImageBytes(base64Chars)} of base64, limit ` +
+                                        `${formatImageBytes(MAX_INLINE_IMAGE_BASE64_CHARS)}) and backend hosting ` +
+                                        `${routing ? "failed" : "is not configured"}. ` +
+                                        "Retry with format=jpg, a smaller window, or once image hosting is available.",
+                                },
+                            ],
+                            details: {
+                                format,
+                                sizeBytes: buf.length,
+                                platform: process.platform,
+                                savedPath,
+                                hosted: false,
+                                inlineTooLarge: true,
+                                isError: true,
+                            },
+                        };
+                    }
+                    imageBlock = fitted;
+                }
                 return {
                     content: [
-                        { type: "text", text: `✓ ${label} (${format.toUpperCase()}, ${(buf.length / 1024).toFixed(1)} KB).\nSaved: ${savedPath}\nThe image is attached and will be analyzed by the vision model.` },
-                        { type: "image", data: base64, mimeType },
+                        {
+                            type: "text",
+                            text: `✓ ${label} (${format.toUpperCase()}, ${(buf.length / 1024).toFixed(1)} KB).\n` +
+                                `Saved: ${savedPath}` +
+                                (hostedUrl ? `\nHosted: ${hostedUrl}` : "") +
+                                (coordMeta?.metadata ? `\n\n${coordMeta.metadata}` : "") +
+                                "\nThe image is attached and will be analyzed by the vision model.",
+                        },
+                        imageBlock,
                     ],
-                    details: { format, sizeBytes: buf.length, platform: process.platform, savedPath },
+                    details: {
+                        format,
+                        sizeBytes: buf.length,
+                        platform: process.platform,
+                        savedPath,
+                        ...(hostedUrl ? { url: hostedUrl, hosted: true } : { hosted: false }),
+                        ...(coordMeta?.details ?? {}),
+                    },
                 };
             };
             const makeError = (msg) => ({
@@ -509,11 +849,17 @@ export default async function desktopScreenshotExtension(ext) {
                             details: { isError: true, searched: p.windowTitle },
                         };
                     }
-                    const buf = await screenshotWindow(match.id, { format }, process.platform);
-                    return makeImageResult(buf, `Window "${match.title}" (${match.owner})`);
+                    const raw = await screenshotWindow(match.id, { format }, process.platform);
+                    const coordMeta = process.platform === "darwin"
+                        ? await normalizeDarwinScreenshot(raw, format, "window")
+                        : undefined;
+                    return await makeImageResult(coordMeta?.buffer ?? raw, `Window "${match.title}" (${match.owner})`, coordMeta);
                 }
-                const buf = await screenshot({ format });
-                return makeImageResult(buf, "Desktop screenshot");
+                const raw = await screenshot({ format });
+                const coordMeta = process.platform === "darwin"
+                    ? await normalizeDarwinScreenshot(raw, format, "screen")
+                    : undefined;
+                return await makeImageResult(coordMeta?.buffer ?? raw, "Desktop screenshot", coordMeta);
             }
             catch (err) {
                 return makeError(err instanceof Error ? err.message : String(err));

package/dist/extensions/image-generation/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/extensions/image-generation/index.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAmB,YAAY,EAAkB,MAAM,iCAAiC,CAAC;AAyLrG,wBAA8B,wBAAwB,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAwDvF"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/extensions/image-generation/index.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAmB,YAAY,EAAkB,MAAM,iCAAiC,CAAC;AA0QrG,wBAA8B,wBAAwB,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiGvF"}