@pi-unipi/image 2.2.0

package/README.md

@@ -0,0 +1,82 @@
+# @pi-unipi/image
+
+Image generation and image recognition tools for the agent.
+
+## Tools
+
+| Tool | Description |
+|------|-------------|
+| `image_generate` | Generate an image from a text prompt. Returned inline and saved to disk. |
+| `image_recognize` | Analyze an image with a vision model. Accepts a file path, `data:` URL, or base64. |
+
+## Commands
+
+| Command | Description |
+|---------|-------------|
+| `/unipi:image-settings` | Configure models, output directory and the recognition system prompt |
+
+## image_generate
+
+```
+image_generate(prompt: "A cutaway diagram of a submarine, technical illustration")
+image_generate(prompt: "...", model: "flux.2-pro")
+```
+
+Models come from pi-ai's image catalog — 34 models including FLUX.2,
+Gemini 3 Pro Image, GPT-5 Image, Recraft and Riverflow — all served through
+**OpenRouter**, so an OpenRouter key is required
+([get one](https://openrouter.ai/keys)).
+
+The `model` parameter is fuzzy-matched, so `flux`, `recraft` and
+`gemini-3-pro` all work. Omit it to use the model chosen in
+`/unipi:image-settings`.
+
+Images are returned inline **and** written to `~/.unipi/images` by default;
+the saved path is reported back to the agent. A failed write never discards a
+successfully generated image.
+
+## image_recognize
+
+```
+image_recognize(image: "./screenshot.png")
+image_recognize(image: "./error.png", prompt: "What does the stack trace say?")
+```
+
+Uses any chat model whose input modality includes `image`. A model that cannot
+accept images is rejected up front with a clear message rather than failing
+inside the provider.
+
+Input can be a local file path, a `data:` URL, or raw base64. The media type is
+detected from the file's magic numbers, so a `.jpg` that is really a PNG still
+works. Supported: PNG, JPEG, GIF, WebP. Remote URLs are not fetched.
+
+Prefer file paths — inlining base64 into the conversation is far more
+expensive in tokens.
+
+## Configuration
+
+`~/.unipi/config/image/config.json`:
+
+```json
+{
+  "generate": {
+    "enabled": true,
+    "model": "openrouter/google/gemini-3-pro-image",
+    "outputDir": "~/.unipi/images",
+    "saveToDisk": true
+  },
+  "recognize": {
+    "enabled": true,
+    "model": "",
+    "systemPrompt": "You are a precise image analyst…"
+  }
+}
+```
+
+- `recognize.model` empty means "use the session's current model".
+- The system prompt is fully customizable, and can be overridden per call.
+- Enabling or disabling a tool takes effect next session, since tools are
+  registered at startup.
+- Every read falls back to defaults, so a corrupt config never breaks the tools.
+
+Set `UNIPI_IMAGE_CONFIG_DIR` to relocate the config directory (used by tests).

package/index.ts

@@ -0,0 +1 @@
+export { default } from "./src/index.js";

package/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "@pi-unipi/image",
+  "version": "2.2.0",
+  "description": "Image generation and image recognition tools for the Pi coding agent",
+  "type": "module",
+  "main": "src/index.ts",
+  "license": "MIT",
+  "author": "Neuron Mr White",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Neuron-Mr-White/unipi.git",
+    "directory": "packages/image"
+  },
+  "homepage": "https://github.com/Neuron-Mr-White/unipi#readme",
+  "bugs": {
+    "url": "https://github.com/Neuron-Mr-White/unipi/issues"
+  },
+  "keywords": [
+    "pi-package",
+    "pi-extension",
+    "pi-coding-agent",
+    "unipi",
+    "image",
+    "vision",
+    "image-generation"
+  ],
+  "files": [
+    "index.ts",
+    "src/**/*.ts",
+    "skills/**/*",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@pi-unipi/core": "2.2.0"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-ai": "^0.80.0",
+    "@earendil-works/pi-coding-agent": "^0.80.0",
+    "@earendil-works/pi-tui": "^0.80.0",
+    "typebox": "^1.1.38"
+  },
+  "devDependencies": {
+    "@types/node": "^25.6.0"
+  },
+  "scripts": {
+    "test": "npx tsx --test tests/**/*.test.ts"
+  },
+  "pi": {
+    "extensions": [],
+    "skills": [],
+    "prompts": [],
+    "themes": []
+  }
+}

package/skills/image/SKILL.md

@@ -0,0 +1,73 @@
+---
+name: image
+description: Generate images from text prompts and analyze images with vision models
+---
+
+# Image Tools
+
+Two agent tools: `image_generate` creates images from a text prompt, and
+`image_recognize` analyzes an existing image.
+
+## image_generate
+
+```
+image_generate(prompt: "A cutaway diagram of a submarine, technical illustration, muted blues")
+image_generate(prompt: "...", model: "flux.2-pro")
+```
+
+- `prompt` (required) — describe subject, style, composition and lighting.
+  Detail materially improves the result.
+- `model` (optional) — fuzzy-matched against the image catalog
+  (`flux`, `gemini-3-pro-image`, `recraft-v4`, …). Omit to use the model
+  configured in `/unipi:image-settings`.
+
+The image is returned inline and, when `saveToDisk` is on (the default),
+written to the output directory (default `~/.unipi/images`). The saved path is
+reported in the result.
+
+**Image generation costs money per call.** Never regenerate an image
+speculatively — only when the user asks for a change.
+
+Models are served through OpenRouter, so an OpenRouter key is required:
+https://openrouter.ai/keys
+
+## image_recognize
+
+```
+image_recognize(image: "./screenshot.png")
+image_recognize(image: "./error.png", prompt: "What does the stack trace say?")
+image_recognize(image: "...", model: "claude-sonnet", systemPrompt: "Reply only with the visible text.")
+```
+
+- `image` (required) — a local file path, a `data:` URL, or raw base64.
+  **Prefer a file path**: inlining base64 into the conversation is far more
+  expensive. Remote URLs are not fetched — download first.
+- `prompt` (optional) — the question to ask. Defaults to a general
+  description. A specific question gives a far more useful answer.
+- `model` (optional) — must accept image input. Omit to use the configured
+  model, falling back to the session's current model.
+- `systemPrompt` (optional) — override the configured system prompt for one
+  call.
+
+Supported types: PNG, JPEG, GIF, WebP. The type is detected from the file's
+magic numbers, so a misnamed extension still works.
+
+### When to use it
+
+- Reading a screenshot of an error, a stack trace, or failing UI
+- Understanding a design mockup or wireframe before implementing it
+- Extracting content from an architecture diagram or flowchart
+- Checking what a rendered page or chart actually looks like
+
+## Configuration
+
+`/unipi:image-settings` configures both tools:
+
+- Generation model (picker over the image catalog)
+- Recognition model (picker over vision-capable models only)
+- Enable/disable either tool
+- Output directory and whether to save to disk
+- The recognition system prompt
+
+Config lives at `~/.unipi/config/image/config.json`. Toggling a tool on or off
+takes effect on the next session, since tools are registered at startup.

package/src/commands.ts

@@ -0,0 +1,17 @@
+/**
+ * @pi-unipi/image — Slash commands
+ */
+
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { IMAGE_COMMANDS, UNIPI_PREFIX } from "@pi-unipi/core";
+
+import { showSettingsDialog } from "./tui/settings-dialog.js";
+
+export function registerImageCommands(pi: ExtensionAPI): void {
+  pi.registerCommand(`${UNIPI_PREFIX}${IMAGE_COMMANDS.SETTINGS}`, {
+    description: "Configure image generation and recognition models",
+    handler: async (_args, ctx) => {
+      await showSettingsDialog(ctx);
+    },
+  });
+}

package/src/generate.ts

@@ -0,0 +1,201 @@
+/**
+ * @pi-unipi/image — Image generation
+ *
+ * Wraps pi-ai's image API. `generateImages` never rejects — failures come back
+ * as `stopReason: "error"` — so every call site must inspect the result rather
+ * than relying on try/catch.
+ */
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+import { getImagesModels, type ImageGenModel, type ImagesModelsLike } from "./models.js";
+
+export interface GeneratedImage {
+  /** Base64 image data. */
+  data: string;
+  mimeType: string;
+  /** Absolute path, when saved to disk. */
+  path?: string;
+}
+
+export interface GenerateResult {
+  images: GeneratedImage[];
+  /** Any accompanying commentary from the model. */
+  text: string;
+  model: string;
+  provider: string;
+}
+
+/** Structural view of pi-ai's AssistantImages. */
+interface AssistantImagesLike {
+  output?: Array<{ type?: string; text?: string; data?: string; mimeType?: string }>;
+  stopReason?: string;
+  errorMessage?: string;
+  usage?: unknown;
+}
+
+/** Injectable generation function, for tests. */
+export type GenerateImagesFn = (
+  model: unknown,
+  context: { input: Array<{ type: string; text?: string }> },
+  options: { apiKey?: string; signal?: AbortSignal },
+) => Promise<AssistantImagesLike>;
+
+/** Extension for a media type, for naming saved files. */
+function extensionFor(mimeType: string): string {
+  switch (mimeType) {
+    case "image/png":
+      return ".png";
+    case "image/jpeg":
+      return ".jpg";
+    case "image/webp":
+      return ".webp";
+    case "image/gif":
+      return ".gif";
+    case "image/svg+xml":
+      return ".svg";
+    default:
+      return ".img";
+  }
+}
+
+/** Filesystem-safe slug from a prompt, for a recognizable filename. */
+export function slugify(prompt: string, maxLength = 40): string {
+  const slug = prompt
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, maxLength)
+    .replace(/-+$/g, "");
+  return slug || "image";
+}
+
+/** Build a collision-free filename. */
+export function buildFileName(
+  prompt: string,
+  mimeType: string,
+  index: number,
+  now: Date = new Date(),
+): string {
+  const stamp = now.toISOString().replace(/[:.]/g, "-").replace("T", "_").slice(0, 19);
+  const suffix = index > 0 ? `-${index + 1}` : "";
+  return `${stamp}-${slugify(prompt)}${suffix}${extensionFor(mimeType)}`;
+}
+
+/**
+ * Write an image to disk.
+ * @returns the absolute path, or undefined if the write failed (never throws —
+ * a failed save must not discard a successfully generated image).
+ */
+export function saveImage(
+  outputDir: string,
+  fileName: string,
+  base64: string,
+): string | undefined {
+  try {
+    fs.mkdirSync(outputDir, { recursive: true });
+    const target = path.join(outputDir, fileName);
+    fs.writeFileSync(target, Buffer.from(base64, "base64"));
+    return target;
+  } catch {
+    return undefined;
+  }
+}
+
+export interface GenerateOptions {
+  prompt: string;
+  model: ImageGenModel;
+  /** Fallback key, used only when pi-ai's own auth resolution comes up empty. */
+  apiKey?: string;
+  signal?: AbortSignal;
+  /** Absolute directory for saved images; omit to skip saving. */
+  outputDir?: string;
+  now?: Date;
+  /** Injected images collection, for tests. */
+  images?: ImagesModelsLike;
+}
+
+/**
+ * Generate images and optionally save them.
+ * @throws {Error} with an actionable message when generation fails.
+ */
+export async function generateImage(options: GenerateOptions): Promise<GenerateResult> {
+  const { prompt, model, signal, outputDir, now } = options;
+
+  if (!prompt.trim()) {
+    throw new Error("A non-empty prompt is required.");
+  }
+
+  const imagesApi = options.images ?? (await getImagesModels());
+  if (!imagesApi) {
+    throw new Error(
+      "Image generation is unavailable — this version of pi-ai does not expose an image API.",
+    );
+  }
+
+  // Prefer pi-ai's own credential store, then the caller-supplied fallback so
+  // a bare OPENROUTER_API_KEY still works.
+  let apiKey: string | undefined;
+  try {
+    apiKey = (await imagesApi.getAuth(model))?.apiKey;
+  } catch {
+    // Reported as a missing key below.
+  }
+  apiKey ||= options.apiKey;
+
+  if (!apiKey) {
+    throw new Error(
+      `No API key for provider "${model.provider}".\n` +
+        `→ Add one with /login, or set the provider's API key environment variable.\n` +
+        `→ Image models are served through OpenRouter: https://openrouter.ai/keys`,
+    );
+  }
+
+  const result = (await imagesApi.generateImages(
+    model,
+    { input: [{ type: "text", text: prompt }] },
+    { apiKey, ...(signal ? { signal } : {}) },
+  )) as AssistantImagesLike;
+
+  // pi-ai reports failures in-band rather than rejecting.
+  if (result.stopReason === "error") {
+    throw new Error(result.errorMessage || "Image generation failed.");
+  }
+  if (result.stopReason === "aborted") {
+    throw new Error("Image generation was cancelled.");
+  }
+
+  const images: GeneratedImage[] = [];
+  const textParts: string[] = [];
+
+  for (const part of result.output ?? []) {
+    if (part?.type === "image" && typeof part.data === "string" && part.data.length > 0) {
+      images.push({ data: part.data, mimeType: part.mimeType || "image/png" });
+    } else if (part?.type === "text" && typeof part.text === "string" && part.text.trim()) {
+      textParts.push(part.text.trim());
+    }
+  }
+
+  if (images.length === 0) {
+    throw new Error(
+      textParts.length > 0
+        ? `The model returned no image. It said: ${textParts.join(" ")}`
+        : "The model returned no image.",
+    );
+  }
+
+  if (outputDir) {
+    images.forEach((image, index) => {
+      const fileName = buildFileName(prompt, image.mimeType, index, now);
+      image.path = saveImage(outputDir, fileName, image.data);
+    });
+  }
+
+  return {
+    images,
+    text: textParts.join("\n"),
+    model: model.id,
+    provider: model.provider,
+  };
+}

package/src/image-source.ts

@@ -0,0 +1,204 @@
+/**
+ * @pi-unipi/image — Image input handling
+ *
+ * Accepts a local file path, a data: URL, or a raw base64 string and
+ * normalizes it to the `{ data, mimeType }` shape pi-ai expects.
+ */
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+export interface LoadedImage {
+  /** Base64-encoded image data (no data: prefix). */
+  data: string;
+  /** IANA media type. */
+  mimeType: string;
+  /** Where it came from, for the tool's result message. */
+  source: "file" | "data-url" | "base64";
+  /** Absolute path when loaded from disk. */
+  path?: string;
+}
+
+/** Media types accepted by the vision APIs. */
+export const SUPPORTED_MIME_TYPES = [
+  "image/png",
+  "image/jpeg",
+  "image/gif",
+  "image/webp",
+] as const;
+
+const EXTENSION_MIME: Record<string, string> = {
+  ".png": "image/png",
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".jfif": "image/jpeg",
+  ".gif": "image/gif",
+  ".webp": "image/webp",
+};
+
+/** Magic-number signatures, checked before trusting a file extension. */
+const SIGNATURES: Array<{ mimeType: string; test: (b: Buffer) => boolean }> = [
+  {
+    mimeType: "image/png",
+    test: (b) =>
+      b.length >= 8 &&
+      b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47 &&
+      b[4] === 0x0d && b[5] === 0x0a && b[6] === 0x1a && b[7] === 0x0a,
+  },
+  {
+    mimeType: "image/jpeg",
+    test: (b) => b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff,
+  },
+  {
+    mimeType: "image/gif",
+    test: (b) => b.length >= 6 && b.subarray(0, 6).toString("ascii").startsWith("GIF8"),
+  },
+  {
+    mimeType: "image/webp",
+    test: (b) =>
+      b.length >= 12 &&
+      b.subarray(0, 4).toString("ascii") === "RIFF" &&
+      b.subarray(8, 12).toString("ascii") === "WEBP",
+  },
+];
+
+/** Infer a media type from a file extension. */
+export function mimeTypeFromExtension(filePath: string): string | undefined {
+  return EXTENSION_MIME[path.extname(filePath).toLowerCase()];
+}
+
+/** Detect a media type from magic numbers. Authoritative over the extension. */
+export function detectMimeType(buffer: Buffer): string | undefined {
+  return SIGNATURES.find((s) => s.test(buffer))?.mimeType;
+}
+
+/** Whether a media type is accepted by the vision APIs. */
+export function isSupportedMimeType(mimeType: string): boolean {
+  return (SUPPORTED_MIME_TYPES as readonly string[]).includes(mimeType);
+}
+
+/** Parse a `data:image/png;base64,...` URL. */
+export function parseDataUrl(
+  input: string,
+): { data: string; mimeType: string } | null {
+  const match = input.match(/^data:([^;,]+)(;[^,]*)?,(.*)$/s);
+  if (!match) return null;
+  const [, mimeType, params, payload] = match;
+  if (!params?.includes("base64")) return null;
+  return { data: payload.trim(), mimeType: mimeType.trim().toLowerCase() };
+}
+
+/** Whether a string plausibly is raw base64 (and long enough to be an image). */
+export function looksLikeBase64(input: string): boolean {
+  const compact = input.replace(/\s/g, "");
+  if (compact.length < 64) return false;
+  return /^[A-Za-z0-9+/]+={0,2}$/.test(compact);
+}
+
+function describeUnsupported(mimeType: string): string {
+  return (
+    `Unsupported image type "${mimeType}". ` +
+    `Supported types: ${SUPPORTED_MIME_TYPES.join(", ")}.`
+  );
+}
+
+/**
+ * Resolve an `image` parameter to base64 data plus a media type.
+ *
+ * Order: data: URL, then existing file path, then raw base64. A path that
+ * looks like a path but does not exist reports the missing file rather than
+ * being misread as base64.
+ *
+ * @throws {Error} with an actionable message on any unusable input.
+ */
+export function loadImage(input: string, cwd = process.cwd()): LoadedImage {
+  const trimmed = input.trim();
+  if (!trimmed) {
+    throw new Error("No image provided. Pass a file path, a data: URL, or base64 data.");
+  }
+
+  // 1. data: URL
+  if (trimmed.startsWith("data:")) {
+    const parsed = parseDataUrl(trimmed);
+    if (!parsed) {
+      throw new Error("Malformed data: URL — expected data:<mime>;base64,<data>.");
+    }
+    if (!isSupportedMimeType(parsed.mimeType)) {
+      throw new Error(describeUnsupported(parsed.mimeType));
+    }
+    return { data: parsed.data, mimeType: parsed.mimeType, source: "data-url" };
+  }
+
+  // 2. Remote URLs are not fetched — be explicit rather than silently failing.
+  if (/^https?:\/\//i.test(trimmed)) {
+    throw new Error(
+      "Remote image URLs are not supported. " +
+        "Download the image first, then pass the local file path.",
+    );
+  }
+
+  // 3. File path
+  const looksLikePath =
+    trimmed.startsWith("/") ||
+    trimmed.startsWith("~") ||
+    trimmed.startsWith(".") ||
+    /[\\/]/.test(trimmed) ||
+    Boolean(mimeTypeFromExtension(trimmed));
+
+  if (looksLikePath) {
+    const resolved = path.isAbsolute(trimmed) ? trimmed : path.resolve(cwd, trimmed);
+
+    if (!fs.existsSync(resolved)) {
+      throw new Error(`Image file not found: ${resolved}`);
+    }
+    if (!fs.statSync(resolved).isFile()) {
+      throw new Error(`Not a file: ${resolved}`);
+    }
+
+    const buffer = fs.readFileSync(resolved);
+    if (buffer.length === 0) {
+      throw new Error(`Image file is empty: ${resolved}`);
+    }
+
+    // Trust the content over the extension.
+    const mimeType = detectMimeType(buffer) ?? mimeTypeFromExtension(resolved);
+    if (!mimeType) {
+      throw new Error(
+        `Could not determine the image type of ${resolved}. ` +
+          `Supported types: ${SUPPORTED_MIME_TYPES.join(", ")}.`,
+      );
+    }
+    if (!isSupportedMimeType(mimeType)) {
+      throw new Error(describeUnsupported(mimeType));
+    }
+
+    return {
+      data: buffer.toString("base64"),
+      mimeType,
+      source: "file",
+      path: resolved,
+    };
+  }
+
+  // 4. Raw base64
+  if (looksLikeBase64(trimmed)) {
+    const compact = trimmed.replace(/\s/g, "");
+    const mimeType = detectMimeType(Buffer.from(compact, "base64"));
+    if (!mimeType) {
+      throw new Error(
+        "Could not determine the image type of the supplied base64 data. " +
+          "Prefer a file path, or use a data: URL that declares the media type.",
+      );
+    }
+    return { data: compact, mimeType, source: "base64" };
+  }
+
+  throw new Error(
+    `Could not interpret "${truncate(trimmed)}" as an image. ` +
+      "Pass a local file path, a data: URL, or base64-encoded image data.",
+  );
+}
+
+function truncate(value: string, max = 60): string {
+  return value.length <= max ? value : `${value.slice(0, max)}…`;
+}