@mariozechner/pi-coding-agent 0.45.7 → 0.47.0

package/dist/utils/image-resize.js

@@ -1,4 +1,4 @@
-import { getVips } from "./vips.js";
+import photon from "@silvia-odwyer/photon-node";
 // 4.5MB - provides headroom below Anthropic's 5MB limit
 const DEFAULT_MAX_BYTES = 4.5 * 1024 * 1024;
 const DEFAULT_OPTIONS = {
@@ -15,8 +15,8 @@ function pickSmaller(a, b) {
  * Resize an image to fit within the specified max dimensions and file size.
  * Returns the original image if it already fits within the limits.
  *
- * Uses wasm-vips for image processing. If wasm-vips is not available (e.g., in some
- * environments), returns the original image unchanged.
+ * Uses Photon (Rust/WASM) for image processing. If Photon is not available,
+ * returns the original image unchanged.
  *
  * Strategy for staying under maxBytes:
  * 1. First resize to maxWidth/maxHeight
@@ -26,107 +26,57 @@ function pickSmaller(a, b) {
  */
 export async function resizeImage(img, options) {
     const opts = { ...DEFAULT_OPTIONS, ...options };
-    const buffer = Buffer.from(img.data, "base64");
-    const vipsOrNull = await getVips();
-    if (!vipsOrNull) {
-        // wasm-vips not available - return original image
-        // We can't get dimensions without vips, so return 0s
-        return {
-            data: img.data,
-            mimeType: img.mimeType,
-            originalWidth: 0,
-            originalHeight: 0,
-            width: 0,
-            height: 0,
-            wasResized: false,
-        };
-    }
-    // Capture non-null reference for use in nested functions
-    const vips = vipsOrNull;
-    // Load image to get metadata
-    let sourceImg;
+    const inputBuffer = Buffer.from(img.data, "base64");
+    let image;
     try {
-        sourceImg = vips.Image.newFromBuffer(buffer);
-    }
-    catch {
-        // Failed to load image
-        return {
-            data: img.data,
-            mimeType: img.mimeType,
-            originalWidth: 0,
-            originalHeight: 0,
-            width: 0,
-            height: 0,
-            wasResized: false,
-        };
-    }
-    const originalWidth = sourceImg.width;
-    const originalHeight = sourceImg.height;
-    // Check if already within all limits (dimensions AND size)
-    const originalSize = buffer.length;
-    if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && originalSize <= opts.maxBytes) {
-        sourceImg.delete();
+        image = photon.PhotonImage.new_from_byteslice(new Uint8Array(inputBuffer));
+        const originalWidth = image.get_width();
+        const originalHeight = image.get_height();
         const format = img.mimeType?.split("/")[1] ?? "png";
-        return {
-            data: img.data,
-            mimeType: img.mimeType ?? `image/${format}`,
-            originalWidth,
-            originalHeight,
-            width: originalWidth,
-            height: originalHeight,
-            wasResized: false,
-        };
-    }
-    // Calculate initial dimensions respecting max limits
-    let targetWidth = originalWidth;
-    let targetHeight = originalHeight;
-    if (targetWidth > opts.maxWidth) {
-        targetHeight = Math.round((targetHeight * opts.maxWidth) / targetWidth);
-        targetWidth = opts.maxWidth;
-    }
-    if (targetHeight > opts.maxHeight) {
-        targetWidth = Math.round((targetWidth * opts.maxHeight) / targetHeight);
-        targetHeight = opts.maxHeight;
-    }
-    // Helper to resize and encode in both formats, returning the smaller one
-    function tryBothFormats(width, height, jpegQuality) {
-        // Load image fresh and resize using scale factor
-        // (Using newFromBuffer + resize instead of thumbnailBuffer to avoid lazy re-read issues)
-        const img = vips.Image.newFromBuffer(buffer);
-        const scale = Math.min(width / img.width, height / img.height);
-        const resized = scale < 1 ? img.resize(scale) : img;
-        const pngBuffer = resized.writeToBuffer(".png");
-        const jpegBuffer = resized.writeToBuffer(".jpg", { Q: jpegQuality });
-        if (resized !== img) {
-            resized.delete();
+        // Check if already within all limits (dimensions AND size)
+        const originalSize = inputBuffer.length;
+        if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && originalSize <= opts.maxBytes) {
+            return {
+                data: img.data,
+                mimeType: img.mimeType ?? `image/${format}`,
+                originalWidth,
+                originalHeight,
+                width: originalWidth,
+                height: originalHeight,
+                wasResized: false,
+            };
         }
-        img.delete();
-        return pickSmaller({ buffer: pngBuffer, mimeType: "image/png" }, { buffer: jpegBuffer, mimeType: "image/jpeg" });
-    }
-    // Clean up the source image
-    sourceImg.delete();
-    // Try to produce an image under maxBytes
-    const qualitySteps = [85, 70, 55, 40];
-    const scaleSteps = [1.0, 0.75, 0.5, 0.35, 0.25];
-    let best;
-    let finalWidth = targetWidth;
-    let finalHeight = targetHeight;
-    // First attempt: resize to target dimensions, try both formats
-    best = tryBothFormats(targetWidth, targetHeight, opts.jpegQuality);
-    if (best.buffer.length <= opts.maxBytes) {
-        return {
-            data: Buffer.from(best.buffer).toString("base64"),
-            mimeType: best.mimeType,
-            originalWidth,
-            originalHeight,
-            width: finalWidth,
-            height: finalHeight,
-            wasResized: true,
-        };
-    }
-    // Still too large - try JPEG with decreasing quality (and compare to PNG each time)
-    for (const quality of qualitySteps) {
-        best = tryBothFormats(targetWidth, targetHeight, quality);
+        // Calculate initial dimensions respecting max limits
+        let targetWidth = originalWidth;
+        let targetHeight = originalHeight;
+        if (targetWidth > opts.maxWidth) {
+            targetHeight = Math.round((targetHeight * opts.maxWidth) / targetWidth);
+            targetWidth = opts.maxWidth;
+        }
+        if (targetHeight > opts.maxHeight) {
+            targetWidth = Math.round((targetWidth * opts.maxHeight) / targetHeight);
+            targetHeight = opts.maxHeight;
+        }
+        // Helper to resize and encode in both formats, returning the smaller one
+        function tryBothFormats(width, height, jpegQuality) {
+            const resized = photon.resize(image, width, height, photon.SamplingFilter.Lanczos3);
+            try {
+                const pngBuffer = resized.get_bytes();
+                const jpegBuffer = resized.get_bytes_jpeg(jpegQuality);
+                return pickSmaller({ buffer: pngBuffer, mimeType: "image/png" }, { buffer: jpegBuffer, mimeType: "image/jpeg" });
+            }
+            finally {
+                resized.free();
+            }
+        }
+        // Try to produce an image under maxBytes
+        const qualitySteps = [85, 70, 55, 40];
+        const scaleSteps = [1.0, 0.75, 0.5, 0.35, 0.25];
+        let best;
+        let finalWidth = targetWidth;
+        let finalHeight = targetHeight;
+        // First attempt: resize to target dimensions, try both formats
+        best = tryBothFormats(targetWidth, targetHeight, opts.jpegQuality);
         if (best.buffer.length <= opts.maxBytes) {
             return {
                 data: Buffer.from(best.buffer).toString("base64"),
@@ -138,17 +88,9 @@ export async function resizeImage(img, options) {
                 wasResized: true,
             };
         }
-    }
-    // Still too large - reduce dimensions progressively
-    for (const scale of scaleSteps) {
-        finalWidth = Math.round(targetWidth * scale);
-        finalHeight = Math.round(targetHeight * scale);
-        // Skip if dimensions are too small
-        if (finalWidth < 100 || finalHeight < 100) {
-            break;
-        }
+        // Still too large - try JPEG with decreasing quality
         for (const quality of qualitySteps) {
-            best = tryBothFormats(finalWidth, finalHeight, quality);
+            best = tryBothFormats(targetWidth, targetHeight, quality);
             if (best.buffer.length <= opts.maxBytes) {
                 return {
                     data: Buffer.from(best.buffer).toString("base64"),
@@ -161,18 +103,56 @@ export async function resizeImage(img, options) {
                 };
             }
         }
+        // Still too large - reduce dimensions progressively
+        for (const scale of scaleSteps) {
+            finalWidth = Math.round(targetWidth * scale);
+            finalHeight = Math.round(targetHeight * scale);
+            if (finalWidth < 100 || finalHeight < 100) {
+                break;
+            }
+            for (const quality of qualitySteps) {
+                best = tryBothFormats(finalWidth, finalHeight, quality);
+                if (best.buffer.length <= opts.maxBytes) {
+                    return {
+                        data: Buffer.from(best.buffer).toString("base64"),
+                        mimeType: best.mimeType,
+                        originalWidth,
+                        originalHeight,
+                        width: finalWidth,
+                        height: finalHeight,
+                        wasResized: true,
+                    };
+                }
+            }
+        }
+        // Last resort: return smallest version we produced
+        return {
+            data: Buffer.from(best.buffer).toString("base64"),
+            mimeType: best.mimeType,
+            originalWidth,
+            originalHeight,
+            width: finalWidth,
+            height: finalHeight,
+            wasResized: true,
+        };
+    }
+    catch {
+        // Failed to load image
+        return {
+            data: img.data,
+            mimeType: img.mimeType,
+            originalWidth: 0,
+            originalHeight: 0,
+            width: 0,
+            height: 0,
+            wasResized: false,
+        };
+    }
+    finally {
+        if (image) {
+            image.free();
+        }
     }
-    // Last resort: return smallest version we produced even if over limit
-    // (the API will reject it, but at least we tried everything)
-    return {
-        data: Buffer.from(best.buffer).toString("base64"),
-        mimeType: best.mimeType,
-        originalWidth,
-        originalHeight,
-        width: finalWidth,
-        height: finalHeight,
-        wasResized: true,
-    };
 }
 /**
  * Format a dimension note for resized images.

package/dist/utils/image-resize.js.map

@@ -1 +1 @@
-{"version":3,"file":"image-resize.js","sourceRoot":"","sources":["../../src/utils/image-resize.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAmBpC,wDAAwD;AACxD,MAAM,iBAAiB,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;AAE5C,MAAM,eAAe,GAAiC;IACrD,QAAQ,EAAE,IAAI;IACd,SAAS,EAAE,IAAI;IACf,QAAQ,EAAE,iBAAiB;IAC3B,WAAW,EAAE,EAAE;CACf,CAAC;AAEF,gDAAgD;AAChD,SAAS,WAAW,CACnB,CAA2C,EAC3C,CAA2C,EACA;IAC3C,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAAA,CAClD;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAiB,EAAE,OAA4B,EAAyB;IACzG,MAAM,IAAI,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAC;IAChD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAE/C,MAAM,UAAU,GAAG,MAAM,OAAO,EAAE,CAAC;IACnC,IAAI,CAAC,UAAU,EAAE,CAAC;QACjB,kDAAkD;QAClD,qDAAqD;QACrD,OAAO;YACN,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,CAAC;YAChB,cAAc,EAAE,CAAC;YACjB,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,KAAK;SACjB,CAAC;IACH,CAAC;IACD,yDAAyD;IACzD,MAAM,IAAI,GAAG,UAAU,CAAC;IAExB,6BAA6B;IAC7B,IAAI,SAA0C,CAAC;IAC/C,IAAI,CAAC;QACJ,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACR,uBAAuB;QACvB,OAAO;YACN,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,CAAC;YAChB,cAAc,EAAE,CAAC;YACjB,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,KAAK;SACjB,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,SAAS,CAAC,KAAK,CAAC;IACtC,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,CAAC;IAExC,2DAA2D;IAC3D,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC;IACnC,IAAI,aAAa,IAAI,IAAI,CAAC,QAAQ,IAAI,cAAc,IAAI,IAAI,CAAC,SAAS,IAAI,YAAY,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACzG,SAAS,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;QACpD,OAAO;YACN,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS,MAAM,EAAE;YAC3C,aAAa;YACb,cAAc;YACd,KAAK,EAAE,aAAa;YACpB,MAAM,EAAE,cAAc;YACtB,UAAU,EAAE,KAAK;SACjB,CAAC;IACH,CAAC;IAED,qDAAqD;IACrD,IAAI,WAAW,GAAG,aAAa,CAAC;IAChC,IAAI,YAAY,GAAG,cAAc,CAAC;IAElC,IAAI,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QACjC,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,WAAW,CAAC,CAAC;QACxE,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC;IAC7B,CAAC;IACD,IAAI,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QACnC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,YAAY,CAAC,CAAC;QACxE,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC;IAC/B,CAAC;IAED,yEAAyE;IACzE,SAAS,cAAc,CACtB,KAAa,EACb,MAAc,EACd,WAAmB,EACwB;QAC3C,iDAAiD;QACjD,yFAAyF;QACzF,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAEpD,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QAChD,MAAM,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC;QAErE,IAAI,OAAO,KAAK,GAAG,EAAE,CAAC;YACrB,OAAO,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;QACD,GAAG,CAAC,MAAM,EAAE,CAAC;QAEb,OAAO,WAAW,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC;IAAA,CACjH;IAED,4BAA4B;IAC5B,SAAS,CAAC,MAAM,EAAE,CAAC;IAEnB,yCAAyC;IACzC,MAAM,YAAY,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;IACtC,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IAEhD,IAAI,IAA8C,CAAC;IACnD,IAAI,UAAU,GAAG,WAAW,CAAC;IAC7B,IAAI,WAAW,GAAG,YAAY,CAAC;IAE/B,+DAA+D;IAC/D,IAAI,GAAG,cAAc,CAAC,WAAW,EAAE,YAAY,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAEnE,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACzC,OAAO;YACN,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACjD,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa;YACb,cAAc;YACd,KAAK,EAAE,UAAU;YACjB,MAAM,EAAE,WAAW;YACnB,UAAU,EAAE,IAAI;SAChB,CAAC;IACH,CAAC;IAED,oFAAoF;IACpF,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;QACpC,IAAI,GAAG,cAAc,CAAC,WAAW,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;QAE1D,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACzC,OAAO;gBACN,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBACjD,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,aAAa;gBACb,cAAc;gBACd,KAAK,EAAE,UAAU;gBACjB,MAAM,EAAE,WAAW;gBACnB,UAAU,EAAE,IAAI;aAChB,CAAC;QACH,CAAC;IACF,CAAC;IAED,oDAAoD;IACpD,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAChC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,CAAC;QAC7C,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,CAAC;QAE/C,mCAAmC;QACnC,IAAI,UAAU,GAAG,GAAG,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;YAC3C,MAAM;QACP,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACpC,IAAI,GAAG,cAAc,CAAC,UAAU,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;YAExD,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACzC,OAAO;oBACN,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBACjD,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,aAAa;oBACb,cAAc;oBACd,KAAK,EAAE,UAAU;oBACjB,MAAM,EAAE,WAAW;oBACnB,UAAU,EAAE,IAAI;iBAChB,CAAC;YACH,CAAC;QACF,CAAC;IACF,CAAC;IAED,sEAAsE;IACtE,6DAA6D;IAC7D,OAAO;QACN,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACjD,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,aAAa;QACb,cAAc;QACd,KAAK,EAAE,UAAU;QACjB,MAAM,EAAE,WAAW;QACnB,UAAU,EAAE,IAAI;KAChB,CAAC;AAAA,CACF;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAAoB,EAAsB;IAC7E,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC;IAClD,OAAO,oBAAoB,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,cAAc,kBAAkB,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,6BAA6B,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,6BAA6B,CAAC;AAAA,CAClM","sourcesContent":["import type { ImageContent } from \"@mariozechner/pi-ai\";\nimport { getVips } from \"./vips.js\";\n\nexport interface ImageResizeOptions {\n\tmaxWidth?: number; // Default: 2000\n\tmaxHeight?: number; // Default: 2000\n\tmaxBytes?: number; // Default: 4.5MB (below Anthropic's 5MB limit)\n\tjpegQuality?: number; // Default: 80\n}\n\nexport interface ResizedImage {\n\tdata: string; // base64\n\tmimeType: string;\n\toriginalWidth: number;\n\toriginalHeight: number;\n\twidth: number;\n\theight: number;\n\twasResized: boolean;\n}\n\n// 4.5MB - provides headroom below Anthropic's 5MB limit\nconst DEFAULT_MAX_BYTES = 4.5 * 1024 * 1024;\n\nconst DEFAULT_OPTIONS: Required<ImageResizeOptions> = {\n\tmaxWidth: 2000,\n\tmaxHeight: 2000,\n\tmaxBytes: DEFAULT_MAX_BYTES,\n\tjpegQuality: 80,\n};\n\n/** Helper to pick the smaller of two buffers */\nfunction pickSmaller(\n\ta: { buffer: Uint8Array; mimeType: string },\n\tb: { buffer: Uint8Array; mimeType: string },\n): { buffer: Uint8Array; mimeType: string } {\n\treturn a.buffer.length <= b.buffer.length ? a : b;\n}\n\n/**\n * Resize an image to fit within the specified max dimensions and file size.\n * Returns the original image if it already fits within the limits.\n *\n * Uses wasm-vips for image processing. If wasm-vips is not available (e.g., in some\n * environments), returns the original image unchanged.\n *\n * Strategy for staying under maxBytes:\n * 1. First resize to maxWidth/maxHeight\n * 2. Try both PNG and JPEG formats, pick the smaller one\n * 3. If still too large, try JPEG with decreasing quality\n * 4. If still too large, progressively reduce dimensions\n */\nexport async function resizeImage(img: ImageContent, options?: ImageResizeOptions): Promise<ResizedImage> {\n\tconst opts = { ...DEFAULT_OPTIONS, ...options };\n\tconst buffer = Buffer.from(img.data, \"base64\");\n\n\tconst vipsOrNull = await getVips();\n\tif (!vipsOrNull) {\n\t\t// wasm-vips not available - return original image\n\t\t// We can't get dimensions without vips, so return 0s\n\t\treturn {\n\t\t\tdata: img.data,\n\t\t\tmimeType: img.mimeType,\n\t\t\toriginalWidth: 0,\n\t\t\toriginalHeight: 0,\n\t\t\twidth: 0,\n\t\t\theight: 0,\n\t\t\twasResized: false,\n\t\t};\n\t}\n\t// Capture non-null reference for use in nested functions\n\tconst vips = vipsOrNull;\n\n\t// Load image to get metadata\n\tlet sourceImg: InstanceType<typeof vips.Image>;\n\ttry {\n\t\tsourceImg = vips.Image.newFromBuffer(buffer);\n\t} catch {\n\t\t// Failed to load image\n\t\treturn {\n\t\t\tdata: img.data,\n\t\t\tmimeType: img.mimeType,\n\t\t\toriginalWidth: 0,\n\t\t\toriginalHeight: 0,\n\t\t\twidth: 0,\n\t\t\theight: 0,\n\t\t\twasResized: false,\n\t\t};\n\t}\n\n\tconst originalWidth = sourceImg.width;\n\tconst originalHeight = sourceImg.height;\n\n\t// Check if already within all limits (dimensions AND size)\n\tconst originalSize = buffer.length;\n\tif (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && originalSize <= opts.maxBytes) {\n\t\tsourceImg.delete();\n\t\tconst format = img.mimeType?.split(\"/\")[1] ?? \"png\";\n\t\treturn {\n\t\t\tdata: img.data,\n\t\t\tmimeType: img.mimeType ?? `image/${format}`,\n\t\t\toriginalWidth,\n\t\t\toriginalHeight,\n\t\t\twidth: originalWidth,\n\t\t\theight: originalHeight,\n\t\t\twasResized: false,\n\t\t};\n\t}\n\n\t// Calculate initial dimensions respecting max limits\n\tlet targetWidth = originalWidth;\n\tlet targetHeight = originalHeight;\n\n\tif (targetWidth > opts.maxWidth) {\n\t\ttargetHeight = Math.round((targetHeight * opts.maxWidth) / targetWidth);\n\t\ttargetWidth = opts.maxWidth;\n\t}\n\tif (targetHeight > opts.maxHeight) {\n\t\ttargetWidth = Math.round((targetWidth * opts.maxHeight) / targetHeight);\n\t\ttargetHeight = opts.maxHeight;\n\t}\n\n\t// Helper to resize and encode in both formats, returning the smaller one\n\tfunction tryBothFormats(\n\t\twidth: number,\n\t\theight: number,\n\t\tjpegQuality: number,\n\t): { buffer: Uint8Array; mimeType: string } {\n\t\t// Load image fresh and resize using scale factor\n\t\t// (Using newFromBuffer + resize instead of thumbnailBuffer to avoid lazy re-read issues)\n\t\tconst img = vips.Image.newFromBuffer(buffer);\n\t\tconst scale = Math.min(width / img.width, height / img.height);\n\t\tconst resized = scale < 1 ? img.resize(scale) : img;\n\n\t\tconst pngBuffer = resized.writeToBuffer(\".png\");\n\t\tconst jpegBuffer = resized.writeToBuffer(\".jpg\", { Q: jpegQuality });\n\n\t\tif (resized !== img) {\n\t\t\tresized.delete();\n\t\t}\n\t\timg.delete();\n\n\t\treturn pickSmaller({ buffer: pngBuffer, mimeType: \"image/png\" }, { buffer: jpegBuffer, mimeType: \"image/jpeg\" });\n\t}\n\n\t// Clean up the source image\n\tsourceImg.delete();\n\n\t// Try to produce an image under maxBytes\n\tconst qualitySteps = [85, 70, 55, 40];\n\tconst scaleSteps = [1.0, 0.75, 0.5, 0.35, 0.25];\n\n\tlet best: { buffer: Uint8Array; mimeType: string };\n\tlet finalWidth = targetWidth;\n\tlet finalHeight = targetHeight;\n\n\t// First attempt: resize to target dimensions, try both formats\n\tbest = tryBothFormats(targetWidth, targetHeight, opts.jpegQuality);\n\n\tif (best.buffer.length <= opts.maxBytes) {\n\t\treturn {\n\t\t\tdata: Buffer.from(best.buffer).toString(\"base64\"),\n\t\t\tmimeType: best.mimeType,\n\t\t\toriginalWidth,\n\t\t\toriginalHeight,\n\t\t\twidth: finalWidth,\n\t\t\theight: finalHeight,\n\t\t\twasResized: true,\n\t\t};\n\t}\n\n\t// Still too large - try JPEG with decreasing quality (and compare to PNG each time)\n\tfor (const quality of qualitySteps) {\n\t\tbest = tryBothFormats(targetWidth, targetHeight, quality);\n\n\t\tif (best.buffer.length <= opts.maxBytes) {\n\t\t\treturn {\n\t\t\t\tdata: Buffer.from(best.buffer).toString(\"base64\"),\n\t\t\t\tmimeType: best.mimeType,\n\t\t\t\toriginalWidth,\n\t\t\t\toriginalHeight,\n\t\t\t\twidth: finalWidth,\n\t\t\t\theight: finalHeight,\n\t\t\t\twasResized: true,\n\t\t\t};\n\t\t}\n\t}\n\n\t// Still too large - reduce dimensions progressively\n\tfor (const scale of scaleSteps) {\n\t\tfinalWidth = Math.round(targetWidth * scale);\n\t\tfinalHeight = Math.round(targetHeight * scale);\n\n\t\t// Skip if dimensions are too small\n\t\tif (finalWidth < 100 || finalHeight < 100) {\n\t\t\tbreak;\n\t\t}\n\n\t\tfor (const quality of qualitySteps) {\n\t\t\tbest = tryBothFormats(finalWidth, finalHeight, quality);\n\n\t\t\tif (best.buffer.length <= opts.maxBytes) {\n\t\t\t\treturn {\n\t\t\t\t\tdata: Buffer.from(best.buffer).toString(\"base64\"),\n\t\t\t\t\tmimeType: best.mimeType,\n\t\t\t\t\toriginalWidth,\n\t\t\t\t\toriginalHeight,\n\t\t\t\t\twidth: finalWidth,\n\t\t\t\t\theight: finalHeight,\n\t\t\t\t\twasResized: true,\n\t\t\t\t};\n\t\t\t}\n\t\t}\n\t}\n\n\t// Last resort: return smallest version we produced even if over limit\n\t// (the API will reject it, but at least we tried everything)\n\treturn {\n\t\tdata: Buffer.from(best.buffer).toString(\"base64\"),\n\t\tmimeType: best.mimeType,\n\t\toriginalWidth,\n\t\toriginalHeight,\n\t\twidth: finalWidth,\n\t\theight: finalHeight,\n\t\twasResized: true,\n\t};\n}\n\n/**\n * Format a dimension note for resized images.\n * This helps the model understand the coordinate mapping.\n */\nexport function formatDimensionNote(result: ResizedImage): string | undefined {\n\tif (!result.wasResized) {\n\t\treturn undefined;\n\t}\n\n\tconst scale = result.originalWidth / result.width;\n\treturn `[Image: original ${result.originalWidth}x${result.originalHeight}, displayed at ${result.width}x${result.height}. Multiply coordinates by ${scale.toFixed(2)} to map to original image.]`;\n}\n"]}
+{"version":3,"file":"image-resize.js","sourceRoot":"","sources":["../../src/utils/image-resize.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,4BAA4B,CAAC;AAmBhD,wDAAwD;AACxD,MAAM,iBAAiB,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;AAE5C,MAAM,eAAe,GAAiC;IACrD,QAAQ,EAAE,IAAI;IACd,SAAS,EAAE,IAAI;IACf,QAAQ,EAAE,iBAAiB;IAC3B,WAAW,EAAE,EAAE;CACf,CAAC;AAEF,gDAAgD;AAChD,SAAS,WAAW,CACnB,CAA2C,EAC3C,CAA2C,EACA;IAC3C,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAAA,CAClD;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAiB,EAAE,OAA4B,EAAyB;IACzG,MAAM,IAAI,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAC;IAChD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAEpD,IAAI,KAA2E,CAAC;IAChF,IAAI,CAAC;QACJ,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,kBAAkB,CAAC,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;QAE3E,MAAM,aAAa,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;QACxC,MAAM,cAAc,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;QAEpD,2DAA2D;QAC3D,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CAAC;QACxC,IAAI,aAAa,IAAI,IAAI,CAAC,QAAQ,IAAI,cAAc,IAAI,IAAI,CAAC,SAAS,IAAI,YAAY,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACzG,OAAO;gBACN,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS,MAAM,EAAE;gBAC3C,aAAa;gBACb,cAAc;gBACd,KAAK,EAAE,aAAa;gBACpB,MAAM,EAAE,cAAc;gBACtB,UAAU,EAAE,KAAK;aACjB,CAAC;QACH,CAAC;QAED,qDAAqD;QACrD,IAAI,WAAW,GAAG,aAAa,CAAC;QAChC,IAAI,YAAY,GAAG,cAAc,CAAC;QAElC,IAAI,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YACjC,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,WAAW,CAAC,CAAC;YACxE,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC7B,CAAC;QACD,IAAI,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;YACnC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,YAAY,CAAC,CAAC;YACxE,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC;QAC/B,CAAC;QAED,yEAAyE;QACzE,SAAS,cAAc,CACtB,KAAa,EACb,MAAc,EACd,WAAmB,EACwB;YAC3C,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,KAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YAErF,IAAI,CAAC;gBACJ,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;gBACtC,MAAM,UAAU,GAAG,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;gBAEvD,OAAO,WAAW,CACjB,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,EAC5C,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,CAC9C,CAAC;YACH,CAAC;oBAAS,CAAC;gBACV,OAAO,CAAC,IAAI,EAAE,CAAC;YAChB,CAAC;QAAA,CACD;QAED,yCAAyC;QACzC,MAAM,YAAY,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QACtC,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAEhD,IAAI,IAA8C,CAAC;QACnD,IAAI,UAAU,GAAG,WAAW,CAAC;QAC7B,IAAI,WAAW,GAAG,YAAY,CAAC;QAE/B,+DAA+D;QAC/D,IAAI,GAAG,cAAc,CAAC,WAAW,EAAE,YAAY,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAEnE,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACzC,OAAO;gBACN,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBACjD,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,aAAa;gBACb,cAAc;gBACd,KAAK,EAAE,UAAU;gBACjB,MAAM,EAAE,WAAW;gBACnB,UAAU,EAAE,IAAI;aAChB,CAAC;QACH,CAAC;QAED,qDAAqD;QACrD,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACpC,IAAI,GAAG,cAAc,CAAC,WAAW,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;YAE1D,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACzC,OAAO;oBACN,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBACjD,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,aAAa;oBACb,cAAc;oBACd,KAAK,EAAE,UAAU;oBACjB,MAAM,EAAE,WAAW;oBACnB,UAAU,EAAE,IAAI;iBAChB,CAAC;YACH,CAAC;QACF,CAAC;QAED,oDAAoD;QACpD,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAChC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,CAAC;YAC7C,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,CAAC;YAE/C,IAAI,UAAU,GAAG,GAAG,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;gBAC3C,MAAM;YACP,CAAC;YAED,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;gBACpC,IAAI,GAAG,cAAc,CAAC,UAAU,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;gBAExD,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACzC,OAAO;wBACN,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;wBACjD,QAAQ,EAAE,IAAI,CAAC,QAAQ;wBACvB,aAAa;wBACb,cAAc;wBACd,KAAK,EAAE,UAAU;wBACjB,MAAM,EAAE,WAAW;wBACnB,UAAU,EAAE,IAAI;qBAChB,CAAC;gBACH,CAAC;YACF,CAAC;QACF,CAAC;QAED,mDAAmD;QACnD,OAAO;YACN,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACjD,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa;YACb,cAAc;YACd,KAAK,EAAE,UAAU;YACjB,MAAM,EAAE,WAAW;YACnB,UAAU,EAAE,IAAI;SAChB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACR,uBAAuB;QACvB,OAAO;YACN,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,CAAC;YAChB,cAAc,EAAE,CAAC;YACjB,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,KAAK;SACjB,CAAC;IACH,CAAC;YAAS,CAAC;QACV,IAAI,KAAK,EAAE,CAAC;YACX,KAAK,CAAC,IAAI,EAAE,CAAC;QACd,CAAC;IACF,CAAC;AAAA,CACD;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAAoB,EAAsB;IAC7E,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC;IAClD,OAAO,oBAAoB,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,cAAc,kBAAkB,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,6BAA6B,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,6BAA6B,CAAC;AAAA,CAClM","sourcesContent":["import type { ImageContent } from \"@mariozechner/pi-ai\";\nimport photon from \"@silvia-odwyer/photon-node\";\n\nexport interface ImageResizeOptions {\n\tmaxWidth?: number; // Default: 2000\n\tmaxHeight?: number; // Default: 2000\n\tmaxBytes?: number; // Default: 4.5MB (below Anthropic's 5MB limit)\n\tjpegQuality?: number; // Default: 80\n}\n\nexport interface ResizedImage {\n\tdata: string; // base64\n\tmimeType: string;\n\toriginalWidth: number;\n\toriginalHeight: number;\n\twidth: number;\n\theight: number;\n\twasResized: boolean;\n}\n\n// 4.5MB - provides headroom below Anthropic's 5MB limit\nconst DEFAULT_MAX_BYTES = 4.5 * 1024 * 1024;\n\nconst DEFAULT_OPTIONS: Required<ImageResizeOptions> = {\n\tmaxWidth: 2000,\n\tmaxHeight: 2000,\n\tmaxBytes: DEFAULT_MAX_BYTES,\n\tjpegQuality: 80,\n};\n\n/** Helper to pick the smaller of two buffers */\nfunction pickSmaller(\n\ta: { buffer: Uint8Array; mimeType: string },\n\tb: { buffer: Uint8Array; mimeType: string },\n): { buffer: Uint8Array; mimeType: string } {\n\treturn a.buffer.length <= b.buffer.length ? a : b;\n}\n\n/**\n * Resize an image to fit within the specified max dimensions and file size.\n * Returns the original image if it already fits within the limits.\n *\n * Uses Photon (Rust/WASM) for image processing. If Photon is not available,\n * returns the original image unchanged.\n *\n * Strategy for staying under maxBytes:\n * 1. First resize to maxWidth/maxHeight\n * 2. Try both PNG and JPEG formats, pick the smaller one\n * 3. If still too large, try JPEG with decreasing quality\n * 4. If still too large, progressively reduce dimensions\n */\nexport async function resizeImage(img: ImageContent, options?: ImageResizeOptions): Promise<ResizedImage> {\n\tconst opts = { ...DEFAULT_OPTIONS, ...options };\n\tconst inputBuffer = Buffer.from(img.data, \"base64\");\n\n\tlet image: ReturnType<typeof photon.PhotonImage.new_from_byteslice> | undefined;\n\ttry {\n\t\timage = photon.PhotonImage.new_from_byteslice(new Uint8Array(inputBuffer));\n\n\t\tconst originalWidth = image.get_width();\n\t\tconst originalHeight = image.get_height();\n\t\tconst format = img.mimeType?.split(\"/\")[1] ?? \"png\";\n\n\t\t// Check if already within all limits (dimensions AND size)\n\t\tconst originalSize = inputBuffer.length;\n\t\tif (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && originalSize <= opts.maxBytes) {\n\t\t\treturn {\n\t\t\t\tdata: img.data,\n\t\t\t\tmimeType: img.mimeType ?? `image/${format}`,\n\t\t\t\toriginalWidth,\n\t\t\t\toriginalHeight,\n\t\t\t\twidth: originalWidth,\n\t\t\t\theight: originalHeight,\n\t\t\t\twasResized: false,\n\t\t\t};\n\t\t}\n\n\t\t// Calculate initial dimensions respecting max limits\n\t\tlet targetWidth = originalWidth;\n\t\tlet targetHeight = originalHeight;\n\n\t\tif (targetWidth > opts.maxWidth) {\n\t\t\ttargetHeight = Math.round((targetHeight * opts.maxWidth) / targetWidth);\n\t\t\ttargetWidth = opts.maxWidth;\n\t\t}\n\t\tif (targetHeight > opts.maxHeight) {\n\t\t\ttargetWidth = Math.round((targetWidth * opts.maxHeight) / targetHeight);\n\t\t\ttargetHeight = opts.maxHeight;\n\t\t}\n\n\t\t// Helper to resize and encode in both formats, returning the smaller one\n\t\tfunction tryBothFormats(\n\t\t\twidth: number,\n\t\t\theight: number,\n\t\t\tjpegQuality: number,\n\t\t): { buffer: Uint8Array; mimeType: string } {\n\t\t\tconst resized = photon.resize(image!, width, height, photon.SamplingFilter.Lanczos3);\n\n\t\t\ttry {\n\t\t\t\tconst pngBuffer = resized.get_bytes();\n\t\t\t\tconst jpegBuffer = resized.get_bytes_jpeg(jpegQuality);\n\n\t\t\t\treturn pickSmaller(\n\t\t\t\t\t{ buffer: pngBuffer, mimeType: \"image/png\" },\n\t\t\t\t\t{ buffer: jpegBuffer, mimeType: \"image/jpeg\" },\n\t\t\t\t);\n\t\t\t} finally {\n\t\t\t\tresized.free();\n\t\t\t}\n\t\t}\n\n\t\t// Try to produce an image under maxBytes\n\t\tconst qualitySteps = [85, 70, 55, 40];\n\t\tconst scaleSteps = [1.0, 0.75, 0.5, 0.35, 0.25];\n\n\t\tlet best: { buffer: Uint8Array; mimeType: string };\n\t\tlet finalWidth = targetWidth;\n\t\tlet finalHeight = targetHeight;\n\n\t\t// First attempt: resize to target dimensions, try both formats\n\t\tbest = tryBothFormats(targetWidth, targetHeight, opts.jpegQuality);\n\n\t\tif (best.buffer.length <= opts.maxBytes) {\n\t\t\treturn {\n\t\t\t\tdata: Buffer.from(best.buffer).toString(\"base64\"),\n\t\t\t\tmimeType: best.mimeType,\n\t\t\t\toriginalWidth,\n\t\t\t\toriginalHeight,\n\t\t\t\twidth: finalWidth,\n\t\t\t\theight: finalHeight,\n\t\t\t\twasResized: true,\n\t\t\t};\n\t\t}\n\n\t\t// Still too large - try JPEG with decreasing quality\n\t\tfor (const quality of qualitySteps) {\n\t\t\tbest = tryBothFormats(targetWidth, targetHeight, quality);\n\n\t\t\tif (best.buffer.length <= opts.maxBytes) {\n\t\t\t\treturn {\n\t\t\t\t\tdata: Buffer.from(best.buffer).toString(\"base64\"),\n\t\t\t\t\tmimeType: best.mimeType,\n\t\t\t\t\toriginalWidth,\n\t\t\t\t\toriginalHeight,\n\t\t\t\t\twidth: finalWidth,\n\t\t\t\t\theight: finalHeight,\n\t\t\t\t\twasResized: true,\n\t\t\t\t};\n\t\t\t}\n\t\t}\n\n\t\t// Still too large - reduce dimensions progressively\n\t\tfor (const scale of scaleSteps) {\n\t\t\tfinalWidth = Math.round(targetWidth * scale);\n\t\t\tfinalHeight = Math.round(targetHeight * scale);\n\n\t\t\tif (finalWidth < 100 || finalHeight < 100) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tfor (const quality of qualitySteps) {\n\t\t\t\tbest = tryBothFormats(finalWidth, finalHeight, quality);\n\n\t\t\t\tif (best.buffer.length <= opts.maxBytes) {\n\t\t\t\t\treturn {\n\t\t\t\t\t\tdata: Buffer.from(best.buffer).toString(\"base64\"),\n\t\t\t\t\t\tmimeType: best.mimeType,\n\t\t\t\t\t\toriginalWidth,\n\t\t\t\t\t\toriginalHeight,\n\t\t\t\t\t\twidth: finalWidth,\n\t\t\t\t\t\theight: finalHeight,\n\t\t\t\t\t\twasResized: true,\n\t\t\t\t\t};\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Last resort: return smallest version we produced\n\t\treturn {\n\t\t\tdata: Buffer.from(best.buffer).toString(\"base64\"),\n\t\t\tmimeType: best.mimeType,\n\t\t\toriginalWidth,\n\t\t\toriginalHeight,\n\t\t\twidth: finalWidth,\n\t\t\theight: finalHeight,\n\t\t\twasResized: true,\n\t\t};\n\t} catch {\n\t\t// Failed to load image\n\t\treturn {\n\t\t\tdata: img.data,\n\t\t\tmimeType: img.mimeType,\n\t\t\toriginalWidth: 0,\n\t\t\toriginalHeight: 0,\n\t\t\twidth: 0,\n\t\t\theight: 0,\n\t\t\twasResized: false,\n\t\t};\n\t} finally {\n\t\tif (image) {\n\t\t\timage.free();\n\t\t}\n\t}\n}\n\n/**\n * Format a dimension note for resized images.\n * This helps the model understand the coordinate mapping.\n */\nexport function formatDimensionNote(result: ResizedImage): string | undefined {\n\tif (!result.wasResized) {\n\t\treturn undefined;\n\t}\n\n\tconst scale = result.originalWidth / result.width;\n\treturn `[Image: original ${result.originalWidth}x${result.originalHeight}, displayed at ${result.width}x${result.height}. Multiply coordinates by ${scale.toFixed(2)} to map to original image.]`;\n}\n"]}

package/docs/extensions.md

@@ -256,6 +256,9 @@ pi starts
       ▼
 user sends prompt ─────────────────────────────────────────┐
   │                                                        │
+  ├─► (extension commands checked first, bypass if found)  │
+  ├─► input (can intercept, transform, or handle)          │
+  ├─► (skill/template expansion if not handled)            │
   ├─► before_agent_start (can inject message, modify system prompt)
   ├─► agent_start                                          │
   │                                                        │
@@ -574,6 +577,54 @@ pi.on("user_bash", (event, ctx) => {
 
 **Examples:** [ssh.ts](../examples/extensions/ssh.ts), [interactive-shell.ts](../examples/extensions/interactive-shell.ts)
 
+### Input Events
+
+#### input
+
+Fired when user input is received, after extension commands are checked but before skill and template expansion. The event sees the raw input text, so `/skill:foo` and `/template` are not yet expanded.
+
+**Processing order:**
+1. Extension commands (`/cmd`) checked first - if found, handler runs and input event is skipped
+2. `input` event fires - can intercept, transform, or handle
+3. If not handled: skill commands (`/skill:name`) expanded to skill content
+4. If not handled: prompt templates (`/template`) expanded to template content
+5. Agent processing begins (`before_agent_start`, etc.)
+
+```typescript
+pi.on("input", async (event, ctx) => {
+  // event.text - raw input (before skill/template expansion)
+  // event.images - attached images, if any
+  // event.source - "interactive" (typed), "rpc" (API), or "extension" (via sendUserMessage)
+
+  // Transform: rewrite input before expansion
+  if (event.text.startsWith("?quick "))
+    return { action: "transform", text: `Respond briefly: ${event.text.slice(7)}` };
+
+  // Handle: respond without LLM (extension shows its own feedback)
+  if (event.text === "ping") {
+    ctx.ui.notify("pong", "info");
+    return { action: "handled" };
+  }
+
+  // Route by source: skip processing for extension-injected messages
+  if (event.source === "extension") return { action: "continue" };
+
+  // Intercept skill commands before expansion
+  if (event.text.startsWith("/skill:")) {
+    // Could transform, block, or let pass through
+  }
+
+  return { action: "continue" };  // Default: pass through to expansion
+});
+```
+
+**Results:**
+- `continue` - pass through unchanged (default if handler returns nothing)
+- `transform` - modify text/images, then continue to expansion
+- `handled` - skip agent entirely (first handler to return this wins)
+
+Transforms chain across handlers. See [input-transform.ts](../examples/extensions/input-transform.ts).
+
 ## ExtensionContext
 
 Every handler receives `ctx: ExtensionContext`:

package/docs/rpc.md

@@ -52,9 +52,9 @@ With images:
 
 If the agent is streaming and no `streamingBehavior` is specified, the command returns an error.
 
-**Extension commands**: If the message is a hook command (e.g., `/mycommand`), it executes immediately even during streaming. Extension commands manage their own LLM interaction via `pi.sendMessage()`.
+**Extension commands**: If the message is an extension command (e.g., `/mycommand`), it executes immediately even during streaming. Extension commands manage their own LLM interaction via `pi.sendMessage()`.
 
-**Prompt templates**: File-based prompt templates (from `.md` files) are expanded before sending/queueing.
+**Input expansion**: Skill commands (`/skill:name`) and prompt templates (`/template`) are expanded before sending/queueing.
 
 Response:
 ```json
@@ -65,7 +65,7 @@ The `images` field is optional. Each image uses `ImageContent` format with base6
 
 #### steer
 
-Queue a steering message to interrupt the agent mid-run. Delivered after current tool execution, remaining tools are skipped. File-based prompt templates are expanded. Extension commands are not allowed (use `prompt` instead).
+Queue a steering message to interrupt the agent mid-run. Delivered after current tool execution, remaining tools are skipped. Skill commands and prompt templates are expanded. Extension commands are not allowed (use `prompt` instead).
 
 ```json
 {"type": "steer", "message": "Stop and do this instead"}
@@ -80,7 +80,7 @@ See [set_steering_mode](#set_steering_mode) for controlling how steering message
 
 #### follow_up
 
-Queue a follow-up message to be processed after the agent finishes. Delivered only when agent has no more tool calls or steering messages. File-based prompt templates are expanded. Extension commands are not allowed (use `prompt` instead).
+Queue a follow-up message to be processed after the agent finishes. Delivered only when agent has no more tool calls or steering messages. Skill commands and prompt templates are expanded. Extension commands are not allowed (use `prompt` instead).
 
 ```json
 {"type": "follow_up", "message": "After you're done, also do this"}
@@ -108,7 +108,7 @@ Response:
 
 #### new_session
 
-Start a fresh session. Can be cancelled by a `session_before_switch` hook.
+Start a fresh session. Can be cancelled by a `session_before_switch` extension event handler.
 
 ```json
 {"type": "new_session"}
@@ -124,7 +124,7 @@ Response:
 {"type": "response", "command": "new_session", "success": true, "data": {"cancelled": false}}
 ```
 
-If a hook cancelled:
+If an extension cancelled:
 ```json
 {"type": "response", "command": "new_session", "success": true, "data": {"cancelled": true}}
 ```
@@ -525,7 +525,7 @@ Response:
 
 #### switch_session
 
-Load a different session file. Can be cancelled by a `before_switch` hook.
+Load a different session file. Can be cancelled by a `session_before_switch` extension event handler.
 
 ```json
 {"type": "switch_session", "sessionPath": "/path/to/session.jsonl"}
@@ -536,14 +536,14 @@ Response:
 {"type": "response", "command": "switch_session", "success": true, "data": {"cancelled": false}}
 ```
 
-If a hook cancelled the switch:
+If an extension cancelled the switch:
 ```json
 {"type": "response", "command": "switch_session", "success": true, "data": {"cancelled": true}}
 ```
 
 #### fork
 
-Create a new fork from a previous user message. Can be cancelled by a `before_fork` hook. Returns the text of the message being forked from.
+Create a new fork from a previous user message. Can be cancelled by a `session_before_fork` extension event handler. Returns the text of the message being forked from.
 
 ```json
 {"type": "fork", "entryId": "abc123"}
@@ -559,7 +559,7 @@ Response:
 }
 ```
 
-If a hook cancelled the fork:
+If an extension cancelled the fork:
 ```json
 {
   "type": "response",
@@ -634,7 +634,7 @@ Events are streamed to stdout as JSON lines during agent operation. Events do NO
 | `auto_compaction_end` | Auto-compaction completes |
 | `auto_retry_start` | Auto-retry begins (after transient error) |
 | `auto_retry_end` | Auto-retry completes (success or final failure) |
-| `hook_error` | Hook threw an error |
+| `extension_error` | Extension threw an error |
 
 ### agent_start
 
@@ -827,14 +827,14 @@ On final failure (max retries exceeded):
 }
 ```
 
-### hook_error
+### extension_error
 
-Emitted when a hook throws an error.
+Emitted when an extension throws an error.
 
 ```json
 {
-  "type": "hook_error",
-  "hookPath": "/path/to/hook.ts",
+  "type": "extension_error",
+  "extensionPath": "/path/to/extension.ts",
   "event": "tool_call",
   "error": "Error message..."
 }

package/docs/tui.md

@@ -26,6 +26,32 @@ interface Component {
 
 The TUI appends a full SGR reset and OSC 8 reset at the end of each rendered line. Styles do not carry across lines. If you emit multi-line text with styling, reapply styles per line or use `wrapTextWithAnsi()` so styles are preserved for each wrapped line.
 
+## Focusable Interface (IME Support)
+
+Components that display a text cursor and need IME (Input Method Editor) support should implement the `Focusable` interface:
+
+```typescript
+import { CURSOR_MARKER, type Component, type Focusable } from "@mariozechner/pi-tui";
+
+class MyInput implements Component, Focusable {
+  focused: boolean = false;  // Set by TUI when focus changes
+  
+  render(width: number): string[] {
+    const marker = this.focused ? CURSOR_MARKER : "";
+    // Emit marker right before the fake cursor
+    return [`> ${beforeCursor}${marker}\x1b[7m${atCursor}\x1b[27m${afterCursor}`];
+  }
+}
+```
+
+When a `Focusable` component has focus, TUI:
+1. Sets `focused = true` on the component
+2. Scans rendered output for `CURSOR_MARKER` (a zero-width APC escape sequence)
+3. Positions the hardware terminal cursor at that location
+4. Shows the hardware cursor
+
+This enables IME candidate windows to appear at the correct position for CJK input methods. The `Editor` and `Input` built-in components already implement this interface.
+
 ## Using Components
 
 **In hooks** via `ctx.ui.custom()`:

package/examples/extensions/input-transform.ts

@@ -0,0 +1,43 @@
+/**
+ * Input Transform Example - demonstrates the `input` event for intercepting user input.
+ *
+ * Start pi with this extension:
+ *   pi -e ./examples/extensions/input-transform.ts
+ *
+ * Then type these inside pi:
+ *   ?quick What is TypeScript?  → "Respond briefly: What is TypeScript?"
+ *   ping                        → "pong" (instant, no LLM)
+ *   time                        → current time (instant, no LLM)
+ */
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+
+export default function (pi: ExtensionAPI) {
+	pi.on("input", async (event, ctx) => {
+		// Source-based logic: skip processing for extension-injected messages
+		if (event.source === "extension") {
+			return { action: "continue" };
+		}
+
+		// Transform: ?quick prefix for brief responses
+		if (event.text.startsWith("?quick ")) {
+			const query = event.text.slice(7).trim();
+			if (!query) {
+				ctx.ui.notify("Usage: ?quick <question>", "warning");
+				return { action: "handled" };
+			}
+			return { action: "transform", text: `Respond briefly in 1-2 sentences: ${query}` };
+		}
+
+		// Handle: instant responses without LLM (extension shows its own feedback)
+		if (event.text.toLowerCase() === "ping") {
+			ctx.ui.notify("pong", "info");
+			return { action: "handled" };
+		}
+		if (event.text.toLowerCase() === "time") {
+			ctx.ui.notify(new Date().toLocaleString(), "info");
+			return { action: "handled" };
+		}
+
+		return { action: "continue" };
+	});
+}

package/examples/extensions/modal-editor.ts

@@ -80,6 +80,6 @@ class ModalEditor extends CustomEditor {
 
 export default function (pi: ExtensionAPI) {
 	pi.on("session_start", (_event, ctx) => {
-		ctx.ui.setEditorComponent((_tui, theme, kb) => new ModalEditor(theme, kb));
+		ctx.ui.setEditorComponent((tui, theme, kb) => new ModalEditor(tui, theme, kb));
 	});
 }

package/examples/extensions/overlay-test.ts

@@ -9,7 +9,7 @@
  */
 
 import type { ExtensionAPI, ExtensionCommandContext, Theme } from "@mariozechner/pi-coding-agent";
-import { matchesKey, visibleWidth } from "@mariozechner/pi-tui";
+import { CURSOR_MARKER, type Focusable, matchesKey, visibleWidth } from "@mariozechner/pi-tui";
 
 export default function (pi: ExtensionAPI) {
 	pi.registerCommand("overlay-test", {
@@ -28,9 +28,12 @@ export default function (pi: ExtensionAPI) {
 	});
 }
 
-class OverlayTestComponent {
+class OverlayTestComponent implements Focusable {
 	readonly width = 70;
 
+	/** Focusable interface - set by TUI when focus changes */
+	focused = false;
+
 	private selected = 0;
 	private items = [
 		{ label: "Search", hasInput: true, text: "", cursor: 0 },
@@ -123,7 +126,9 @@ class OverlayTestComponent {
 					const before = inputDisplay.slice(0, item.cursor);
 					const cursorChar = item.cursor < inputDisplay.length ? inputDisplay[item.cursor] : " ";
 					const after = inputDisplay.slice(item.cursor + 1);
-					inputDisplay = `${before}\x1b[7m${cursorChar}\x1b[27m${after}`;
+					// Emit hardware cursor marker for IME support when focused
+					const marker = this.focused ? CURSOR_MARKER : "";
+					inputDisplay = `${before}${marker}\x1b[7m${cursorChar}\x1b[27m${after}`;
 				}
 				content = `${prefix + label} ${inputDisplay}`;
 			} else {

package/examples/extensions/plan-mode/README.md

@@ -15,7 +15,7 @@ Read-only exploration mode for safe code analysis.
 
 - `/plan` - Toggle plan mode
 - `/todos` - Show current plan progress
-- `Shift+P` - Toggle plan mode (shortcut)
+- `Ctrl+Alt+P` - Toggle plan mode (shortcut)
 
 ## Usage
 

package/examples/extensions/plan-mode/index.ts

@@ -5,7 +5,7 @@
  * When enabled, only read-only tools are available.
  *
  * Features:
- * - /plan command or Shift+P to toggle
+ * - /plan command or Ctrl+Alt+P to toggle
  * - Bash restricted to allowlisted read-only commands
  * - Extracts numbered plan steps from "Plan:" sections
  * - [DONE:n] markers to complete steps during execution
@@ -113,7 +113,7 @@ export default function planModeExtension(pi: ExtensionAPI): void {
 		},
 	});
 
-	pi.registerShortcut(Key.shift("p"), {
+	pi.registerShortcut(Key.ctrlAlt("p"), {
 		description: "Toggle plan mode",
 		handler: async (ctx) => togglePlanMode(ctx),
 	});

package/examples/extensions/question.ts

@@ -90,7 +90,7 @@ export default function question(pi: ExtensionAPI) {
 							noMatch: (t) => theme.fg("warning", t),
 						},
 					};
-					const editor = new Editor(editorTheme);
+					const editor = new Editor(tui, editorTheme);
 
 					editor.onSubmit = (value) => {
 						const trimmed = value.trim();