@mariozechner/pi-coding-agent 0.32.1 → 0.32.3

package/dist/utils/image-resize.js

@@ -1,14 +1,27 @@
+// 4.5MB - provides headroom below Anthropic's 5MB limit
+const DEFAULT_MAX_BYTES = 4.5 * 1024 * 1024;
 const DEFAULT_OPTIONS = {
     maxWidth: 2000,
     maxHeight: 2000,
+    maxBytes: DEFAULT_MAX_BYTES,
     jpegQuality: 80,
 };
+/** Helper to pick the smaller of two buffers */
+function pickSmaller(a, b) {
+    return a.buffer.length <= b.buffer.length ? a : b;
+}
 /**
- * Resize an image to fit within the specified max dimensions.
+ * Resize an image to fit within the specified max dimensions and file size.
  * Returns the original image if it already fits within the limits.
  *
  * Uses sharp for image processing. If sharp is not available (e.g., in some
  * environments), returns the original image unchanged.
+ *
+ * Strategy for staying under maxBytes:
+ * 1. First resize to maxWidth/maxHeight
+ * 2. Try both PNG and JPEG formats, pick the smaller one
+ * 3. If still too large, try JPEG with decreasing quality
+ * 4. If still too large, progressively reduce dimensions
  */
 export async function resizeImage(img, options) {
     const opts = { ...DEFAULT_OPTIONS, ...options };
@@ -32,68 +45,110 @@ export async function resizeImage(img, options) {
     }
     const sharpImg = sharp(buffer);
     const metadata = await sharpImg.metadata();
-    const width = metadata.width ?? 0;
-    const height = metadata.height ?? 0;
+    const originalWidth = metadata.width ?? 0;
+    const originalHeight = metadata.height ?? 0;
     const format = metadata.format ?? img.mimeType?.split("/")[1] ?? "png";
-    // Check if already within limits
-    if (width <= opts.maxWidth && height <= opts.maxHeight) {
+    // Check if already within all limits (dimensions AND size)
+    const originalSize = buffer.length;
+    if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && originalSize <= opts.maxBytes) {
         return {
             data: img.data,
             mimeType: img.mimeType ?? `image/${format}`,
-            originalWidth: width,
-            originalHeight: height,
-            width,
-            height,
+            originalWidth,
+            originalHeight,
+            width: originalWidth,
+            height: originalHeight,
             wasResized: false,
         };
     }
-    // Calculate new dimensions maintaining aspect ratio
-    let newWidth = width;
-    let newHeight = height;
-    if (newWidth > opts.maxWidth) {
-        newHeight = Math.round((newHeight * opts.maxWidth) / newWidth);
-        newWidth = opts.maxWidth;
-    }
-    if (newHeight > opts.maxHeight) {
-        newWidth = Math.round((newWidth * opts.maxHeight) / newHeight);
-        newHeight = opts.maxHeight;
+    // Calculate initial dimensions respecting max limits
+    let targetWidth = originalWidth;
+    let targetHeight = originalHeight;
+    if (targetWidth > opts.maxWidth) {
+        targetHeight = Math.round((targetHeight * opts.maxWidth) / targetWidth);
+        targetWidth = opts.maxWidth;
     }
-    // Resize the image
-    const resized = await sharp(buffer)
-        .resize(newWidth, newHeight, { fit: "inside", withoutEnlargement: true })
-        .toBuffer();
-    // Determine output format - preserve original if possible, otherwise use JPEG
-    let outputMimeType;
-    let outputBuffer;
-    if (format === "jpeg" || format === "jpg") {
-        outputBuffer = await sharp(resized).jpeg({ quality: opts.jpegQuality }).toBuffer();
-        outputMimeType = "image/jpeg";
+    if (targetHeight > opts.maxHeight) {
+        targetWidth = Math.round((targetWidth * opts.maxHeight) / targetHeight);
+        targetHeight = opts.maxHeight;
     }
-    else if (format === "png") {
-        outputBuffer = resized;
-        outputMimeType = "image/png";
+    // Helper to resize and encode in both formats, returning the smaller one
+    async function tryBothFormats(width, height, jpegQuality) {
+        const resized = await sharp(buffer)
+            .resize(width, height, { fit: "inside", withoutEnlargement: true })
+            .toBuffer();
+        const [pngBuffer, jpegBuffer] = await Promise.all([
+            sharp(resized).png({ compressionLevel: 9 }).toBuffer(),
+            sharp(resized).jpeg({ quality: jpegQuality }).toBuffer(),
+        ]);
+        return pickSmaller({ buffer: pngBuffer, mimeType: "image/png" }, { buffer: jpegBuffer, mimeType: "image/jpeg" });
     }
-    else if (format === "gif") {
-        // GIF resize might not preserve animation; convert to PNG for quality
-        outputBuffer = resized;
-        outputMimeType = "image/png";
+    // Try to produce an image under maxBytes
+    const qualitySteps = [85, 70, 55, 40];
+    const scaleSteps = [1.0, 0.75, 0.5, 0.35, 0.25];
+    let best;
+    let finalWidth = targetWidth;
+    let finalHeight = targetHeight;
+    // First attempt: resize to target dimensions, try both formats
+    best = await tryBothFormats(targetWidth, targetHeight, opts.jpegQuality);
+    if (best.buffer.length <= opts.maxBytes) {
+        return {
+            data: best.buffer.toString("base64"),
+            mimeType: best.mimeType,
+            originalWidth,
+            originalHeight,
+            width: finalWidth,
+            height: finalHeight,
+            wasResized: true,
+        };
     }
-    else if (format === "webp") {
-        outputBuffer = resized;
-        outputMimeType = "image/webp";
+    // Still too large - try JPEG with decreasing quality (and compare to PNG each time)
+    for (const quality of qualitySteps) {
+        best = await tryBothFormats(targetWidth, targetHeight, quality);
+        if (best.buffer.length <= opts.maxBytes) {
+            return {
+                data: best.buffer.toString("base64"),
+                mimeType: best.mimeType,
+                originalWidth,
+                originalHeight,
+                width: finalWidth,
+                height: finalHeight,
+                wasResized: true,
+            };
+        }
     }
-    else {
-        // Default to JPEG for unknown formats
-        outputBuffer = await sharp(resized).jpeg({ quality: opts.jpegQuality }).toBuffer();
-        outputMimeType = "image/jpeg";
+    // Still too large - reduce dimensions progressively
+    for (const scale of scaleSteps) {
+        finalWidth = Math.round(targetWidth * scale);
+        finalHeight = Math.round(targetHeight * scale);
+        // Skip if dimensions are too small
+        if (finalWidth < 100 || finalHeight < 100) {
+            break;
+        }
+        for (const quality of qualitySteps) {
+            best = await tryBothFormats(finalWidth, finalHeight, quality);
+            if (best.buffer.length <= opts.maxBytes) {
+                return {
+                    data: best.buffer.toString("base64"),
+                    mimeType: best.mimeType,
+                    originalWidth,
+                    originalHeight,
+                    width: finalWidth,
+                    height: finalHeight,
+                    wasResized: true,
+                };
+            }
+        }
     }
+    // Last resort: return smallest version we produced even if over limit
+    // (the API will reject it, but at least we tried everything)
     return {
-        data: outputBuffer.toString("base64"),
-        mimeType: outputMimeType,
-        originalWidth: width,
-        originalHeight: height,
-        width: newWidth,
-        height: newHeight,
+        data: best.buffer.toString("base64"),
+        mimeType: best.mimeType,
+        originalWidth,
+        originalHeight,
+        width: finalWidth,
+        height: finalHeight,
         wasResized: true,
     };
 }

package/dist/utils/image-resize.js.map

@@ -1 +1 @@
-{"version":3,"file":"image-resize.js","sourceRoot":"","sources":["../../src/utils/image-resize.ts"],"names":[],"mappings":"AAkBA,MAAM,eAAe,GAAiC;IACrD,QAAQ,EAAE,IAAI;IACd,SAAS,EAAE,IAAI;IACf,WAAW,EAAE,EAAE;CACf,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAiB,EAAE,OAA4B,EAAyB;IACzG,MAAM,IAAI,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAC;IAChD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAE/C,IAAI,KAAyC,CAAC;IAC9C,IAAI,CAAC;QACJ,KAAK,GAAG,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACR,8CAA8C;QAC9C,sDAAsD;QACtD,OAAO;YACN,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,CAAC;YAChB,cAAc,EAAE,CAAC;YACjB,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,KAAK;SACjB,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/B,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,CAAC;IAE3C,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;IAClC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,IAAI,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;IAEvE,iCAAiC;IACjC,IAAI,KAAK,IAAI,IAAI,CAAC,QAAQ,IAAI,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;QACxD,OAAO;YACN,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS,MAAM,EAAE;YAC3C,aAAa,EAAE,KAAK;YACpB,cAAc,EAAE,MAAM;YACtB,KAAK;YACL,MAAM;YACN,UAAU,EAAE,KAAK;SACjB,CAAC;IACH,CAAC;IAED,oDAAoD;IACpD,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,SAAS,GAAG,MAAM,CAAC;IAEvB,IAAI,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC9B,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,QAAQ,CAAC,CAAC;QAC/D,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;IAC1B,CAAC;IACD,IAAI,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAChC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,SAAS,CAAC,CAAC;QAC/D,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAC5B,CAAC;IAED,mBAAmB;IACnB,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC;SACjC,MAAM,CAAC,QAAQ,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;SACxE,QAAQ,EAAE,CAAC;IAEb,8EAA8E;IAC9E,IAAI,cAAsB,CAAC;IAC3B,IAAI,YAAoB,CAAC;IAEzB,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QAC3C,YAAY,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;QACnF,cAAc,GAAG,YAAY,CAAC;IAC/B,CAAC;SAAM,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QAC7B,YAAY,GAAG,OAAO,CAAC;QACvB,cAAc,GAAG,WAAW,CAAC;IAC9B,CAAC;SAAM,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QAC7B,sEAAsE;QACtE,YAAY,GAAG,OAAO,CAAC;QACvB,cAAc,GAAG,WAAW,CAAC;IAC9B,CAAC;SAAM,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QAC9B,YAAY,GAAG,OAAO,CAAC;QACvB,cAAc,GAAG,YAAY,CAAC;IAC/B,CAAC;SAAM,CAAC;QACP,sCAAsC;QACtC,YAAY,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;QACnF,cAAc,GAAG,YAAY,CAAC;IAC/B,CAAC;IAED,OAAO;QACN,IAAI,EAAE,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACrC,QAAQ,EAAE,cAAc;QACxB,aAAa,EAAE,KAAK;QACpB,cAAc,EAAE,MAAM;QACtB,KAAK,EAAE,QAAQ;QACf,MAAM,EAAE,SAAS;QACjB,UAAU,EAAE,IAAI;KAChB,CAAC;AAAA,CACF;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAAoB,EAAsB;IAC7E,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC;IAClD,OAAO,oBAAoB,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,cAAc,kBAAkB,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,6BAA6B,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,6BAA6B,CAAC;AAAA,CAClM","sourcesContent":["import type { ImageContent } from \"@mariozechner/pi-ai\";\n\nexport interface ImageResizeOptions {\n\tmaxWidth?: number; // Default: 2000\n\tmaxHeight?: number; // Default: 2000\n\tjpegQuality?: number; // Default: 80\n}\n\nexport interface ResizedImage {\n\tdata: string; // base64\n\tmimeType: string;\n\toriginalWidth: number;\n\toriginalHeight: number;\n\twidth: number;\n\theight: number;\n\twasResized: boolean;\n}\n\nconst DEFAULT_OPTIONS: Required<ImageResizeOptions> = {\n\tmaxWidth: 2000,\n\tmaxHeight: 2000,\n\tjpegQuality: 80,\n};\n\n/**\n * Resize an image to fit within the specified max dimensions.\n * Returns the original image if it already fits within the limits.\n *\n * Uses sharp for image processing. If sharp is not available (e.g., in some\n * environments), returns the original image unchanged.\n */\nexport async function resizeImage(img: ImageContent, options?: ImageResizeOptions): Promise<ResizedImage> {\n\tconst opts = { ...DEFAULT_OPTIONS, ...options };\n\tconst buffer = Buffer.from(img.data, \"base64\");\n\n\tlet sharp: typeof import(\"sharp\") | undefined;\n\ttry {\n\t\tsharp = (await import(\"sharp\")).default;\n\t} catch {\n\t\t// Sharp not available - return original image\n\t\t// We can't get dimensions without sharp, so return 0s\n\t\treturn {\n\t\t\tdata: img.data,\n\t\t\tmimeType: img.mimeType,\n\t\t\toriginalWidth: 0,\n\t\t\toriginalHeight: 0,\n\t\t\twidth: 0,\n\t\t\theight: 0,\n\t\t\twasResized: false,\n\t\t};\n\t}\n\n\tconst sharpImg = sharp(buffer);\n\tconst metadata = await sharpImg.metadata();\n\n\tconst width = metadata.width ?? 0;\n\tconst height = metadata.height ?? 0;\n\tconst format = metadata.format ?? img.mimeType?.split(\"/\")[1] ?? \"png\";\n\n\t// Check if already within limits\n\tif (width <= opts.maxWidth && height <= opts.maxHeight) {\n\t\treturn {\n\t\t\tdata: img.data,\n\t\t\tmimeType: img.mimeType ?? `image/${format}`,\n\t\t\toriginalWidth: width,\n\t\t\toriginalHeight: height,\n\t\t\twidth,\n\t\t\theight,\n\t\t\twasResized: false,\n\t\t};\n\t}\n\n\t// Calculate new dimensions maintaining aspect ratio\n\tlet newWidth = width;\n\tlet newHeight = height;\n\n\tif (newWidth > opts.maxWidth) {\n\t\tnewHeight = Math.round((newHeight * opts.maxWidth) / newWidth);\n\t\tnewWidth = opts.maxWidth;\n\t}\n\tif (newHeight > opts.maxHeight) {\n\t\tnewWidth = Math.round((newWidth * opts.maxHeight) / newHeight);\n\t\tnewHeight = opts.maxHeight;\n\t}\n\n\t// Resize the image\n\tconst resized = await sharp(buffer)\n\t\t.resize(newWidth, newHeight, { fit: \"inside\", withoutEnlargement: true })\n\t\t.toBuffer();\n\n\t// Determine output format - preserve original if possible, otherwise use JPEG\n\tlet outputMimeType: string;\n\tlet outputBuffer: Buffer;\n\n\tif (format === \"jpeg\" || format === \"jpg\") {\n\t\toutputBuffer = await sharp(resized).jpeg({ quality: opts.jpegQuality }).toBuffer();\n\t\toutputMimeType = \"image/jpeg\";\n\t} else if (format === \"png\") {\n\t\toutputBuffer = resized;\n\t\toutputMimeType = \"image/png\";\n\t} else if (format === \"gif\") {\n\t\t// GIF resize might not preserve animation; convert to PNG for quality\n\t\toutputBuffer = resized;\n\t\toutputMimeType = \"image/png\";\n\t} else if (format === \"webp\") {\n\t\toutputBuffer = resized;\n\t\toutputMimeType = \"image/webp\";\n\t} else {\n\t\t// Default to JPEG for unknown formats\n\t\toutputBuffer = await sharp(resized).jpeg({ quality: opts.jpegQuality }).toBuffer();\n\t\toutputMimeType = \"image/jpeg\";\n\t}\n\n\treturn {\n\t\tdata: outputBuffer.toString(\"base64\"),\n\t\tmimeType: outputMimeType,\n\t\toriginalWidth: width,\n\t\toriginalHeight: height,\n\t\twidth: newWidth,\n\t\theight: newHeight,\n\t\twasResized: true,\n\t};\n}\n\n/**\n * Format a dimension note for resized images.\n * This helps the model understand the coordinate mapping.\n */\nexport function formatDimensionNote(result: ResizedImage): string | undefined {\n\tif (!result.wasResized) {\n\t\treturn undefined;\n\t}\n\n\tconst scale = result.originalWidth / result.width;\n\treturn `[Image: original ${result.originalWidth}x${result.originalHeight}, displayed at ${result.width}x${result.height}. Multiply coordinates by ${scale.toFixed(2)} to map to original image.]`;\n}\n"]}
+{"version":3,"file":"image-resize.js","sourceRoot":"","sources":["../../src/utils/image-resize.ts"],"names":[],"mappings":"AAmBA,wDAAwD;AACxD,MAAM,iBAAiB,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;AAE5C,MAAM,eAAe,GAAiC;IACrD,QAAQ,EAAE,IAAI;IACd,SAAS,EAAE,IAAI;IACf,QAAQ,EAAE,iBAAiB;IAC3B,WAAW,EAAE,EAAE;CACf,CAAC;AAEF,gDAAgD;AAChD,SAAS,WAAW,CACnB,CAAuC,EACvC,CAAuC,EACA;IACvC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAAA,CAClD;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAiB,EAAE,OAA4B,EAAyB;IACzG,MAAM,IAAI,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAC;IAChD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAE/C,IAAI,KAAyC,CAAC;IAC9C,IAAI,CAAC;QACJ,KAAK,GAAG,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACR,8CAA8C;QAC9C,sDAAsD;QACtD,OAAO;YACN,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,CAAC;YAChB,cAAc,EAAE,CAAC;YACjB,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,KAAK;SACjB,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/B,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,CAAC;IAE3C,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;IAC1C,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,IAAI,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;IAEvE,2DAA2D;IAC3D,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC;IACnC,IAAI,aAAa,IAAI,IAAI,CAAC,QAAQ,IAAI,cAAc,IAAI,IAAI,CAAC,SAAS,IAAI,YAAY,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACzG,OAAO;YACN,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS,MAAM,EAAE;YAC3C,aAAa;YACb,cAAc;YACd,KAAK,EAAE,aAAa;YACpB,MAAM,EAAE,cAAc;YACtB,UAAU,EAAE,KAAK;SACjB,CAAC;IACH,CAAC;IAED,qDAAqD;IACrD,IAAI,WAAW,GAAG,aAAa,CAAC;IAChC,IAAI,YAAY,GAAG,cAAc,CAAC;IAElC,IAAI,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QACjC,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,WAAW,CAAC,CAAC;QACxE,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC;IAC7B,CAAC;IACD,IAAI,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QACnC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,YAAY,CAAC,CAAC;QACxE,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC;IAC/B,CAAC;IAED,yEAAyE;IACzE,KAAK,UAAU,cAAc,CAC5B,KAAa,EACb,MAAc,EACd,WAAmB,EAC6B;QAChD,MAAM,OAAO,GAAG,MAAM,KAAM,CAAC,MAAM,CAAC;aAClC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;aAClE,QAAQ,EAAE,CAAC;QAEb,MAAM,CAAC,SAAS,EAAE,UAAU,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACjD,KAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;YACvD,KAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC,QAAQ,EAAE;SACzD,CAAC,CAAC;QAEH,OAAO,WAAW,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC;IAAA,CACjH;IAED,yCAAyC;IACzC,MAAM,YAAY,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;IACtC,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IAEhD,IAAI,IAA0C,CAAC;IAC/C,IAAI,UAAU,GAAG,WAAW,CAAC;IAC7B,IAAI,WAAW,GAAG,YAAY,CAAC;IAE/B,+DAA+D;IAC/D,IAAI,GAAG,MAAM,cAAc,CAAC,WAAW,EAAE,YAAY,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAEzE,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACzC,OAAO;YACN,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACpC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa;YACb,cAAc;YACd,KAAK,EAAE,UAAU;YACjB,MAAM,EAAE,WAAW;YACnB,UAAU,EAAE,IAAI;SAChB,CAAC;IACH,CAAC;IAED,oFAAoF;IACpF,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;QACpC,IAAI,GAAG,MAAM,cAAc,CAAC,WAAW,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;QAEhE,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACzC,OAAO;gBACN,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBACpC,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,aAAa;gBACb,cAAc;gBACd,KAAK,EAAE,UAAU;gBACjB,MAAM,EAAE,WAAW;gBACnB,UAAU,EAAE,IAAI;aAChB,CAAC;QACH,CAAC;IACF,CAAC;IAED,oDAAoD;IACpD,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAChC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,CAAC;QAC7C,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,CAAC;QAE/C,mCAAmC;QACnC,IAAI,UAAU,GAAG,GAAG,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;YAC3C,MAAM;QACP,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACpC,IAAI,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;YAE9D,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACzC,OAAO;oBACN,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBACpC,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,aAAa;oBACb,cAAc;oBACd,KAAK,EAAE,UAAU;oBACjB,MAAM,EAAE,WAAW;oBACnB,UAAU,EAAE,IAAI;iBAChB,CAAC;YACH,CAAC;QACF,CAAC;IACF,CAAC;IAED,sEAAsE;IACtE,6DAA6D;IAC7D,OAAO;QACN,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACpC,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,aAAa;QACb,cAAc;QACd,KAAK,EAAE,UAAU;QACjB,MAAM,EAAE,WAAW;QACnB,UAAU,EAAE,IAAI;KAChB,CAAC;AAAA,CACF;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAAoB,EAAsB;IAC7E,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC;IAClD,OAAO,oBAAoB,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,cAAc,kBAAkB,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,6BAA6B,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,6BAA6B,CAAC;AAAA,CAClM","sourcesContent":["import type { ImageContent } from \"@mariozechner/pi-ai\";\n\nexport interface ImageResizeOptions {\n\tmaxWidth?: number; // Default: 2000\n\tmaxHeight?: number; // Default: 2000\n\tmaxBytes?: number; // Default: 4.5MB (below Anthropic's 5MB limit)\n\tjpegQuality?: number; // Default: 80\n}\n\nexport interface ResizedImage {\n\tdata: string; // base64\n\tmimeType: string;\n\toriginalWidth: number;\n\toriginalHeight: number;\n\twidth: number;\n\theight: number;\n\twasResized: boolean;\n}\n\n// 4.5MB - provides headroom below Anthropic's 5MB limit\nconst DEFAULT_MAX_BYTES = 4.5 * 1024 * 1024;\n\nconst DEFAULT_OPTIONS: Required<ImageResizeOptions> = {\n\tmaxWidth: 2000,\n\tmaxHeight: 2000,\n\tmaxBytes: DEFAULT_MAX_BYTES,\n\tjpegQuality: 80,\n};\n\n/** Helper to pick the smaller of two buffers */\nfunction pickSmaller(\n\ta: { buffer: Buffer; mimeType: string },\n\tb: { buffer: Buffer; mimeType: string },\n): { buffer: Buffer; mimeType: string } {\n\treturn a.buffer.length <= b.buffer.length ? a : b;\n}\n\n/**\n * Resize an image to fit within the specified max dimensions and file size.\n * Returns the original image if it already fits within the limits.\n *\n * Uses sharp for image processing. If sharp is not available (e.g., in some\n * environments), returns the original image unchanged.\n *\n * Strategy for staying under maxBytes:\n * 1. First resize to maxWidth/maxHeight\n * 2. Try both PNG and JPEG formats, pick the smaller one\n * 3. If still too large, try JPEG with decreasing quality\n * 4. If still too large, progressively reduce dimensions\n */\nexport async function resizeImage(img: ImageContent, options?: ImageResizeOptions): Promise<ResizedImage> {\n\tconst opts = { ...DEFAULT_OPTIONS, ...options };\n\tconst buffer = Buffer.from(img.data, \"base64\");\n\n\tlet sharp: typeof import(\"sharp\") | undefined;\n\ttry {\n\t\tsharp = (await import(\"sharp\")).default;\n\t} catch {\n\t\t// Sharp not available - return original image\n\t\t// We can't get dimensions without sharp, so return 0s\n\t\treturn {\n\t\t\tdata: img.data,\n\t\t\tmimeType: img.mimeType,\n\t\t\toriginalWidth: 0,\n\t\t\toriginalHeight: 0,\n\t\t\twidth: 0,\n\t\t\theight: 0,\n\t\t\twasResized: false,\n\t\t};\n\t}\n\n\tconst sharpImg = sharp(buffer);\n\tconst metadata = await sharpImg.metadata();\n\n\tconst originalWidth = metadata.width ?? 0;\n\tconst originalHeight = metadata.height ?? 0;\n\tconst format = metadata.format ?? img.mimeType?.split(\"/\")[1] ?? \"png\";\n\n\t// Check if already within all limits (dimensions AND size)\n\tconst originalSize = buffer.length;\n\tif (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && originalSize <= opts.maxBytes) {\n\t\treturn {\n\t\t\tdata: img.data,\n\t\t\tmimeType: img.mimeType ?? `image/${format}`,\n\t\t\toriginalWidth,\n\t\t\toriginalHeight,\n\t\t\twidth: originalWidth,\n\t\t\theight: originalHeight,\n\t\t\twasResized: false,\n\t\t};\n\t}\n\n\t// Calculate initial dimensions respecting max limits\n\tlet targetWidth = originalWidth;\n\tlet targetHeight = originalHeight;\n\n\tif (targetWidth > opts.maxWidth) {\n\t\ttargetHeight = Math.round((targetHeight * opts.maxWidth) / targetWidth);\n\t\ttargetWidth = opts.maxWidth;\n\t}\n\tif (targetHeight > opts.maxHeight) {\n\t\ttargetWidth = Math.round((targetWidth * opts.maxHeight) / targetHeight);\n\t\ttargetHeight = opts.maxHeight;\n\t}\n\n\t// Helper to resize and encode in both formats, returning the smaller one\n\tasync function tryBothFormats(\n\t\twidth: number,\n\t\theight: number,\n\t\tjpegQuality: number,\n\t): Promise<{ buffer: Buffer; mimeType: string }> {\n\t\tconst resized = await sharp!(buffer)\n\t\t\t.resize(width, height, { fit: \"inside\", withoutEnlargement: true })\n\t\t\t.toBuffer();\n\n\t\tconst [pngBuffer, jpegBuffer] = await Promise.all([\n\t\t\tsharp!(resized).png({ compressionLevel: 9 }).toBuffer(),\n\t\t\tsharp!(resized).jpeg({ quality: jpegQuality }).toBuffer(),\n\t\t]);\n\n\t\treturn pickSmaller({ buffer: pngBuffer, mimeType: \"image/png\" }, { buffer: jpegBuffer, mimeType: \"image/jpeg\" });\n\t}\n\n\t// Try to produce an image under maxBytes\n\tconst qualitySteps = [85, 70, 55, 40];\n\tconst scaleSteps = [1.0, 0.75, 0.5, 0.35, 0.25];\n\n\tlet best: { buffer: Buffer; mimeType: string };\n\tlet finalWidth = targetWidth;\n\tlet finalHeight = targetHeight;\n\n\t// First attempt: resize to target dimensions, try both formats\n\tbest = await tryBothFormats(targetWidth, targetHeight, opts.jpegQuality);\n\n\tif (best.buffer.length <= opts.maxBytes) {\n\t\treturn {\n\t\t\tdata: best.buffer.toString(\"base64\"),\n\t\t\tmimeType: best.mimeType,\n\t\t\toriginalWidth,\n\t\t\toriginalHeight,\n\t\t\twidth: finalWidth,\n\t\t\theight: finalHeight,\n\t\t\twasResized: true,\n\t\t};\n\t}\n\n\t// Still too large - try JPEG with decreasing quality (and compare to PNG each time)\n\tfor (const quality of qualitySteps) {\n\t\tbest = await tryBothFormats(targetWidth, targetHeight, quality);\n\n\t\tif (best.buffer.length <= opts.maxBytes) {\n\t\t\treturn {\n\t\t\t\tdata: best.buffer.toString(\"base64\"),\n\t\t\t\tmimeType: best.mimeType,\n\t\t\t\toriginalWidth,\n\t\t\t\toriginalHeight,\n\t\t\t\twidth: finalWidth,\n\t\t\t\theight: finalHeight,\n\t\t\t\twasResized: true,\n\t\t\t};\n\t\t}\n\t}\n\n\t// Still too large - reduce dimensions progressively\n\tfor (const scale of scaleSteps) {\n\t\tfinalWidth = Math.round(targetWidth * scale);\n\t\tfinalHeight = Math.round(targetHeight * scale);\n\n\t\t// Skip if dimensions are too small\n\t\tif (finalWidth < 100 || finalHeight < 100) {\n\t\t\tbreak;\n\t\t}\n\n\t\tfor (const quality of qualitySteps) {\n\t\t\tbest = await tryBothFormats(finalWidth, finalHeight, quality);\n\n\t\t\tif (best.buffer.length <= opts.maxBytes) {\n\t\t\t\treturn {\n\t\t\t\t\tdata: best.buffer.toString(\"base64\"),\n\t\t\t\t\tmimeType: best.mimeType,\n\t\t\t\t\toriginalWidth,\n\t\t\t\t\toriginalHeight,\n\t\t\t\t\twidth: finalWidth,\n\t\t\t\t\theight: finalHeight,\n\t\t\t\t\twasResized: true,\n\t\t\t\t};\n\t\t\t}\n\t\t}\n\t}\n\n\t// Last resort: return smallest version we produced even if over limit\n\t// (the API will reject it, but at least we tried everything)\n\treturn {\n\t\tdata: best.buffer.toString(\"base64\"),\n\t\tmimeType: best.mimeType,\n\t\toriginalWidth,\n\t\toriginalHeight,\n\t\twidth: finalWidth,\n\t\theight: finalHeight,\n\t\twasResized: true,\n\t};\n}\n\n/**\n * Format a dimension note for resized images.\n * This helps the model understand the coordinate mapping.\n */\nexport function formatDimensionNote(result: ResizedImage): string | undefined {\n\tif (!result.wasResized) {\n\t\treturn undefined;\n\t}\n\n\tconst scale = result.originalWidth / result.width;\n\treturn `[Image: original ${result.originalWidth}x${result.originalHeight}, displayed at ${result.width}x${result.height}. Multiply coordinates by ${scale.toFixed(2)} to map to original image.]`;\n}\n"]}

package/docs/rpc.md

@@ -41,6 +41,21 @@ With images:
 {"type": "prompt", "message": "What's in this image?", "images": [{"type": "image", "source": {"type": "base64", "mediaType": "image/png", "data": "..."}}]}
 ```
 
+**During streaming**: If the agent is already streaming, you must specify `streamingBehavior` to queue the message:
+
+```json
+{"type": "prompt", "message": "New instruction", "streamingBehavior": "steer"}
+```
+
+- `"steer"`: Interrupt the agent mid-run. Message is delivered after current tool execution, remaining tools are skipped.
+- `"followUp"`: Wait until the agent finishes. Message is delivered only when agent stops.
+
+If the agent is streaming and no `streamingBehavior` is specified, the command returns an error.
+
+**Hook commands**: If the message is a hook command (e.g., `/mycommand`), it executes immediately even during streaming. Hook commands manage their own LLM interaction via `pi.sendMessage()`.
+
+**Slash commands**: File-based slash commands (from `.md` files) are expanded before sending/queueing.
+
 Response:
 ```json
 {"id": "req-1", "type": "response", "command": "prompt", "success": true}
@@ -48,20 +63,35 @@ Response:
 
 The `images` field is optional. Each image uses `ImageContent` format with base64 or URL source.
 
-#### queue_message
+#### steer
 
-Queue a message to be injected at the next agent turn. Queued messages are added to the conversation without triggering a new prompt. Useful for injecting context mid-conversation.
+Queue a steering message to interrupt the agent mid-run. Delivered after current tool execution, remaining tools are skipped. File-based slash commands are expanded. Hook commands are not allowed (use `prompt` instead).
 
 ```json
-{"type": "queue_message", "message": "Additional context"}
+{"type": "steer", "message": "Stop and do this instead"}
 ```
 
 Response:
 ```json
-{"type": "response", "command": "queue_message", "success": true}
+{"type": "response", "command": "steer", "success": true}
 ```
 
-See [set_queue_mode](#set_queue_mode) for controlling how queued messages are processed.
+See [set_steering_mode](#set_steering_mode) for controlling how steering messages are processed.
+
+#### follow_up
+
+Queue a follow-up message to be processed after the agent finishes. Delivered only when agent has no more tool calls or steering messages. File-based slash commands are expanded. Hook commands are not allowed (use `prompt` instead).
+
+```json
+{"type": "follow_up", "message": "After you're done, also do this"}
+```
+
+Response:
+```json
+{"type": "response", "command": "follow_up", "success": true}
+```
+
+See [set_follow_up_mode](#set_follow_up_mode) for controlling how follow-up messages are processed.
 
 #### abort
 
@@ -120,12 +150,13 @@ Response:
     "thinkingLevel": "medium",
     "isStreaming": false,
     "isCompacting": false,
-    "queueMode": "all",
+    "steeringMode": "all",
+    "followUpMode": "one-at-a-time",
     "sessionFile": "/path/to/session.jsonl",
     "sessionId": "abc123",
     "autoCompactionEnabled": true,
     "messageCount": 5,
-    "queuedMessageCount": 0
+    "pendingMessageCount": 0
   }
 }
 ```
@@ -253,23 +284,40 @@ Response:
 }
 ```
 
-### Queue Mode
+### Queue Modes
+
+#### set_steering_mode
+
+Control how steering messages (from `steer`) are delivered.
+
+```json
+{"type": "set_steering_mode", "mode": "one-at-a-time"}
+```
+
+Modes:
+- `"all"`: Deliver all steering messages at the next interruption point
+- `"one-at-a-time"`: Deliver one steering message per interruption (default)
+
+Response:
+```json
+{"type": "response", "command": "set_steering_mode", "success": true}
+```
 
-#### set_queue_mode
+#### set_follow_up_mode
 
-Control how queued messages (from `queue_message`) are injected into the conversation.
+Control how follow-up messages (from `follow_up`) are delivered.
 
 ```json
-{"type": "set_queue_mode", "mode": "one-at-a-time"}
+{"type": "set_follow_up_mode", "mode": "one-at-a-time"}
 ```
 
 Modes:
-- `"all"`: Inject all queued messages at the next turn
-- `"one-at-a-time"`: Inject one queued message per turn (default)
+- `"all"`: Deliver all follow-up messages when agent finishes
+- `"one-at-a-time"`: Deliver one follow-up message per agent completion (default)
 
 Response:
 ```json
-{"type": "response", "command": "set_queue_mode", "success": true}
+{"type": "response", "command": "set_follow_up_mode", "success": true}
 ```
 
 ### Compaction

package/docs/sdk.md

@@ -77,8 +77,13 @@ The session manages the agent lifecycle, message history, and event streaming.
 ```typescript
 interface AgentSession {
   // Send a prompt and wait for completion
+  // If streaming, requires streamingBehavior option to queue the message
   prompt(text: string, options?: PromptOptions): Promise<void>;
   
+  // Queue messages during streaming
+  steer(text: string): Promise<void>;    // Interrupt: delivered after current tool, skips remaining
+  followUp(text: string): Promise<void>; // Wait: delivered only when agent finishes
+  
   // Subscribe to events (returns unsubscribe function)
   subscribe(listener: (event: AgentSessionEvent) => void): () => void;
   
@@ -122,6 +127,41 @@ interface AgentSession {
 }
 ```
 
+### Prompting and Message Queueing
+
+The `prompt()` method handles slash commands, hook commands, and message sending:
+
+```typescript
+// Basic prompt (when not streaming)
+await session.prompt("What files are here?");
+
+// With images
+await session.prompt("What's in this image?", {
+  images: [{ type: "image", source: { type: "base64", mediaType: "image/png", data: "..." } }]
+});
+
+// During streaming: must specify how to queue the message
+await session.prompt("Stop and do this instead", { streamingBehavior: "steer" });
+await session.prompt("After you're done, also check X", { streamingBehavior: "followUp" });
+```
+
+**Behavior:**
+- **Hook commands** (e.g., `/mycommand`): Execute immediately, even during streaming. They manage their own LLM interaction via `pi.sendMessage()`.
+- **File-based slash commands** (from `.md` files): Expanded to their content before sending/queueing.
+- **During streaming without `streamingBehavior`**: Throws an error. Use `steer()` or `followUp()` directly, or specify the option.
+
+For explicit queueing during streaming:
+
+```typescript
+// Interrupt the agent (delivered after current tool, skips remaining tools)
+await session.steer("New instruction");
+
+// Wait for agent to finish (delivered only when agent stops)
+await session.followUp("After you're done, also do this");
+```
+
+Both `steer()` and `followUp()` expand file-based slash commands but error on hook commands (hook commands cannot be queued).
+
 ### Agent and AgentState
 
 The `Agent` class (from `@mariozechner/pi-agent-core`) handles the core LLM interaction. Access it via `session.agent`.

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@mariozechner/pi-coding-agent",
-	"version": "0.32.1",
+	"version": "0.32.3",
 	"description": "Coding agent CLI with read, bash, edit, write tools and session management",
 	"type": "module",
 	"piConfig": {
@@ -38,9 +38,9 @@
 		"prepublishOnly": "npm run clean && npm run build"
 	},
 	"dependencies": {
-		"@mariozechner/pi-agent-core": "^0.32.1",
-		"@mariozechner/pi-ai": "^0.32.1",
-		"@mariozechner/pi-tui": "^0.32.1",
+		"@mariozechner/pi-agent-core": "^0.32.3",
+		"@mariozechner/pi-ai": "^0.32.3",
+		"@mariozechner/pi-tui": "^0.32.3",
 		"chalk": "^5.5.0",
 		"cli-highlight": "^2.1.11",
 		"diff": "^8.0.2",