@pi-unipi/unipi 2.1.2 → 2.2.0

package/packages/image/src/image-source.ts

@@ -0,0 +1,204 @@
+/**
+ * @pi-unipi/image — Image input handling
+ *
+ * Accepts a local file path, a data: URL, or a raw base64 string and
+ * normalizes it to the `{ data, mimeType }` shape pi-ai expects.
+ */
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+export interface LoadedImage {
+  /** Base64-encoded image data (no data: prefix). */
+  data: string;
+  /** IANA media type. */
+  mimeType: string;
+  /** Where it came from, for the tool's result message. */
+  source: "file" | "data-url" | "base64";
+  /** Absolute path when loaded from disk. */
+  path?: string;
+}
+
+/** Media types accepted by the vision APIs. */
+export const SUPPORTED_MIME_TYPES = [
+  "image/png",
+  "image/jpeg",
+  "image/gif",
+  "image/webp",
+] as const;
+
+const EXTENSION_MIME: Record<string, string> = {
+  ".png": "image/png",
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".jfif": "image/jpeg",
+  ".gif": "image/gif",
+  ".webp": "image/webp",
+};
+
+/** Magic-number signatures, checked before trusting a file extension. */
+const SIGNATURES: Array<{ mimeType: string; test: (b: Buffer) => boolean }> = [
+  {
+    mimeType: "image/png",
+    test: (b) =>
+      b.length >= 8 &&
+      b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47 &&
+      b[4] === 0x0d && b[5] === 0x0a && b[6] === 0x1a && b[7] === 0x0a,
+  },
+  {
+    mimeType: "image/jpeg",
+    test: (b) => b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff,
+  },
+  {
+    mimeType: "image/gif",
+    test: (b) => b.length >= 6 && b.subarray(0, 6).toString("ascii").startsWith("GIF8"),
+  },
+  {
+    mimeType: "image/webp",
+    test: (b) =>
+      b.length >= 12 &&
+      b.subarray(0, 4).toString("ascii") === "RIFF" &&
+      b.subarray(8, 12).toString("ascii") === "WEBP",
+  },
+];
+
+/** Infer a media type from a file extension. */
+export function mimeTypeFromExtension(filePath: string): string | undefined {
+  return EXTENSION_MIME[path.extname(filePath).toLowerCase()];
+}
+
+/** Detect a media type from magic numbers. Authoritative over the extension. */
+export function detectMimeType(buffer: Buffer): string | undefined {
+  return SIGNATURES.find((s) => s.test(buffer))?.mimeType;
+}
+
+/** Whether a media type is accepted by the vision APIs. */
+export function isSupportedMimeType(mimeType: string): boolean {
+  return (SUPPORTED_MIME_TYPES as readonly string[]).includes(mimeType);
+}
+
+/** Parse a `data:image/png;base64,...` URL. */
+export function parseDataUrl(
+  input: string,
+): { data: string; mimeType: string } | null {
+  const match = input.match(/^data:([^;,]+)(;[^,]*)?,(.*)$/s);
+  if (!match) return null;
+  const [, mimeType, params, payload] = match;
+  if (!params?.includes("base64")) return null;
+  return { data: payload.trim(), mimeType: mimeType.trim().toLowerCase() };
+}
+
+/** Whether a string plausibly is raw base64 (and long enough to be an image). */
+export function looksLikeBase64(input: string): boolean {
+  const compact = input.replace(/\s/g, "");
+  if (compact.length < 64) return false;
+  return /^[A-Za-z0-9+/]+={0,2}$/.test(compact);
+}
+
+function describeUnsupported(mimeType: string): string {
+  return (
+    `Unsupported image type "${mimeType}". ` +
+    `Supported types: ${SUPPORTED_MIME_TYPES.join(", ")}.`
+  );
+}
+
+/**
+ * Resolve an `image` parameter to base64 data plus a media type.
+ *
+ * Order: data: URL, then existing file path, then raw base64. A path that
+ * looks like a path but does not exist reports the missing file rather than
+ * being misread as base64.
+ *
+ * @throws {Error} with an actionable message on any unusable input.
+ */
+export function loadImage(input: string, cwd = process.cwd()): LoadedImage {
+  const trimmed = input.trim();
+  if (!trimmed) {
+    throw new Error("No image provided. Pass a file path, a data: URL, or base64 data.");
+  }
+
+  // 1. data: URL
+  if (trimmed.startsWith("data:")) {
+    const parsed = parseDataUrl(trimmed);
+    if (!parsed) {
+      throw new Error("Malformed data: URL — expected data:<mime>;base64,<data>.");
+    }
+    if (!isSupportedMimeType(parsed.mimeType)) {
+      throw new Error(describeUnsupported(parsed.mimeType));
+    }
+    return { data: parsed.data, mimeType: parsed.mimeType, source: "data-url" };
+  }
+
+  // 2. Remote URLs are not fetched — be explicit rather than silently failing.
+  if (/^https?:\/\//i.test(trimmed)) {
+    throw new Error(
+      "Remote image URLs are not supported. " +
+        "Download the image first, then pass the local file path.",
+    );
+  }
+
+  // 3. File path
+  const looksLikePath =
+    trimmed.startsWith("/") ||
+    trimmed.startsWith("~") ||
+    trimmed.startsWith(".") ||
+    /[\\/]/.test(trimmed) ||
+    Boolean(mimeTypeFromExtension(trimmed));
+
+  if (looksLikePath) {
+    const resolved = path.isAbsolute(trimmed) ? trimmed : path.resolve(cwd, trimmed);
+
+    if (!fs.existsSync(resolved)) {
+      throw new Error(`Image file not found: ${resolved}`);
+    }
+    if (!fs.statSync(resolved).isFile()) {
+      throw new Error(`Not a file: ${resolved}`);
+    }
+
+    const buffer = fs.readFileSync(resolved);
+    if (buffer.length === 0) {
+      throw new Error(`Image file is empty: ${resolved}`);
+    }
+
+    // Trust the content over the extension.
+    const mimeType = detectMimeType(buffer) ?? mimeTypeFromExtension(resolved);
+    if (!mimeType) {
+      throw new Error(
+        `Could not determine the image type of ${resolved}. ` +
+          `Supported types: ${SUPPORTED_MIME_TYPES.join(", ")}.`,
+      );
+    }
+    if (!isSupportedMimeType(mimeType)) {
+      throw new Error(describeUnsupported(mimeType));
+    }
+
+    return {
+      data: buffer.toString("base64"),
+      mimeType,
+      source: "file",
+      path: resolved,
+    };
+  }
+
+  // 4. Raw base64
+  if (looksLikeBase64(trimmed)) {
+    const compact = trimmed.replace(/\s/g, "");
+    const mimeType = detectMimeType(Buffer.from(compact, "base64"));
+    if (!mimeType) {
+      throw new Error(
+        "Could not determine the image type of the supplied base64 data. " +
+          "Prefer a file path, or use a data: URL that declares the media type.",
+      );
+    }
+    return { data: compact, mimeType, source: "base64" };
+  }
+
+  throw new Error(
+    `Could not interpret "${truncate(trimmed)}" as an image. ` +
+      "Pass a local file path, a data: URL, or base64-encoded image data.",
+  );
+}
+
+function truncate(value: string, max = 60): string {
+  return value.length <= max ? value : `${value.slice(0, max)}…`;
+}

package/packages/image/src/index.ts

@@ -0,0 +1,99 @@
+/**
+ * @pi-unipi/image — Extension entry
+ *
+ * Provides the `image_generate` and `image_recognize` agent tools plus the
+ * `/unipi:image-settings` command.
+ */
+
+import { dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import {
+  IMAGE_COMMANDS,
+  IMAGE_TOOLS,
+  MODULES,
+  UNIPI_EVENTS,
+  UNIPI_PREFIX,
+  emitEvent,
+  getPackageVersion,
+} from "@pi-unipi/core";
+
+import { registerImageCommands } from "./commands.js";
+import { registerImageTools } from "./tools.js";
+import { listImageGenModels, listVisionModels, type ChatModelRegistry } from "./models.js";
+import { loadConfig } from "./settings.js";
+
+const VERSION = getPackageVersion(dirname(fileURLToPath(import.meta.url)));
+
+/** Info-screen registry, read off the global to avoid load-order coupling. */
+function getInfoRegistry() {
+  return (
+    globalThis as {
+      __unipi_info_registry?: {
+        registerGroup(group: unknown): void;
+      };
+    }
+  ).__unipi_info_registry;
+}
+
+export default function (pi: ExtensionAPI) {
+  registerImageTools(pi);
+  registerImageCommands(pi);
+
+  pi.on("session_start", async (_event, ctx) => {
+    const config = loadConfig();
+
+    const tools: string[] = [];
+    if (config.generate.enabled) tools.push(IMAGE_TOOLS.GENERATE);
+    if (config.recognize.enabled) tools.push(IMAGE_TOOLS.RECOGNIZE);
+
+    emitEvent(pi, UNIPI_EVENTS.MODULE_READY, {
+      name: MODULES.IMAGE,
+      version: VERSION,
+      commands: [`${UNIPI_PREFIX}${IMAGE_COMMANDS.SETTINGS}`],
+      tools,
+    });
+
+    const registry = getInfoRegistry();
+    if (!registry) return;
+
+    registry.registerGroup({
+      id: "image",
+      name: "Image",
+      icon: "🎨",
+      priority: 55,
+      config: {
+        showByDefault: true,
+        stats: [
+          { id: "generate", label: "Generate", show: true },
+          { id: "recognize", label: "Recognize", show: true },
+          { id: "visionModels", label: "Vision Models", show: true },
+        ],
+      },
+      dataProvider: async () => {
+        const current = loadConfig();
+
+        const genModels = await listImageGenModels();
+        const generate = current.generate.enabled
+          ? genModels.length > 0
+            ? current.generate.model
+            : "No image models available"
+          : "Disabled";
+
+        const chatRegistry = (ctx as unknown as { modelRegistry?: ChatModelRegistry })
+          .modelRegistry;
+        const vision = chatRegistry ? listVisionModels(chatRegistry) : [];
+
+        const recognize = current.recognize.enabled
+          ? current.recognize.model || "Session model"
+          : "Disabled";
+
+        return {
+          generate: { value: generate },
+          recognize: { value: recognize },
+          visionModels: { value: String(vision.length) },
+        };
+      },
+    });
+  });
+}

package/packages/image/src/models.ts

@@ -0,0 +1,290 @@
+/**
+ * @pi-unipi/image — Model discovery and resolution
+ *
+ * Two different model families are involved:
+ *
+ * - **Image generation** uses pi-ai's `ImagesModel` catalog, which
+ *   pi-coding-agent does not expose on `ExtensionContext` — it is reached
+ *   through the pi-ai subpath exports.
+ * - **Image recognition** uses ordinary chat models filtered to those whose
+ *   `input` modality includes `"image"` (`ctx.modelRegistry`).
+ */
+
+/** A generation model, kept structural to avoid deep pi-ai type coupling. */
+export interface ImageGenModel {
+  id: string;
+  name?: string;
+  provider: string;
+  api: string;
+  baseUrl?: string;
+  input?: string[];
+  output?: string[];
+  [key: string]: unknown;
+}
+
+/** A vision-capable chat model. */
+export interface VisionModel {
+  id: string;
+  name?: string;
+  provider: string;
+  input?: string[];
+}
+
+/** Minimal chat-model registry surface (pi's ModelRegistry). */
+export interface ChatModelRegistry {
+  find(provider: string, modelId: string): unknown;
+  getAll(): unknown[];
+  getAvailable?(): unknown[];
+  getApiKeyForProvider?(provider: string): Promise<string | undefined>;
+}
+
+/** Split "provider/model-id" — the model id may itself contain slashes. */
+export function splitModelRef(ref: string): { provider: string; id: string } | null {
+  const trimmed = ref.trim();
+  const slash = trimmed.indexOf("/");
+  if (slash <= 0 || slash === trimmed.length - 1) return null;
+  return { provider: trimmed.slice(0, slash), id: trimmed.slice(slash + 1) };
+}
+
+/** Format a model as "provider/model-id". */
+export function formatModelRef(model: { provider: string; id: string }): string {
+  return `${model.provider}/${model.id}`;
+}
+
+/**
+ * pi-ai's runtime image-model collection: model catalog, auth resolution and
+ * generation in one object. Only the parts used here are typed.
+ */
+export interface ImagesModelsLike {
+  getModels(provider?: string): readonly ImageGenModel[];
+  getModel(provider: string, id: string): ImageGenModel | undefined;
+  getAuth(model: ImageGenModel): Promise<{ apiKey?: string } | undefined>;
+  generateImages(
+    model: ImageGenModel,
+    context: { input: Array<{ type: string; text?: string }> },
+    options?: { apiKey?: string; signal?: AbortSignal },
+  ): Promise<unknown>;
+}
+
+let cachedImagesModels: ImagesModelsLike | null = null;
+let imagesModelsAttempted = false;
+
+/**
+ * Load pi-ai's built-in images collection.
+ *
+ * `getImageModels`/`generateImages` are not re-exported from the pi-ai package
+ * root, but `providers/all` exports `builtinImagesModels()`, which is the
+ * supported entry point and also resolves auth. A failure is not fatal: it
+ * degrades to null and callers report that no models are available.
+ */
+export async function getImagesModels(): Promise<ImagesModelsLike | null> {
+  if (cachedImagesModels || imagesModelsAttempted) return cachedImagesModels;
+  imagesModelsAttempted = true;
+
+  try {
+    const mod = (await import("@earendil-works/pi-ai/providers/all")) as unknown as {
+      builtinImagesModels?: () => ImagesModelsLike;
+    };
+    if (typeof mod.builtinImagesModels === "function") {
+      cachedImagesModels = mod.builtinImagesModels();
+    }
+  } catch {
+    cachedImagesModels = null;
+  }
+
+  return cachedImagesModels;
+}
+
+/** List available image-generation models. Empty when unavailable. */
+export async function listImageGenModels(): Promise<ImageGenModel[]> {
+  const images = await getImagesModels();
+  if (!images) return [];
+  try {
+    return [...images.getModels()];
+  } catch {
+    return [];
+  }
+}
+
+/** Inject a stub images collection. Test-only. */
+export function __setImagesModelsForTests(models: ImagesModelsLike | null): void {
+  cachedImagesModels = models;
+  imagesModelsAttempted = models !== null;
+}
+
+/** Reset the model cache. Test-only. */
+export function __resetModelCacheForTests(): void {
+  cachedImagesModels = null;
+  imagesModelsAttempted = false;
+}
+
+/**
+ * Resolve a generation-model reference against the catalog.
+ *
+ * Exact "provider/id" first, then a scored fuzzy match so "flux" or
+ * "gemini-3-pro" work. Returns an error string (not a throw) so the tool can
+ * surface it as a normal tool error listing the alternatives.
+ */
+export function resolveImageGenModel(
+  input: string,
+  models: ImageGenModel[],
+): ImageGenModel | string {
+  const query = input.trim().toLowerCase();
+  if (!query) return "No image model specified.";
+  if (models.length === 0) {
+    return (
+      "No image generation models are available.\n" +
+      "→ Image generation requires an OpenRouter account: https://openrouter.ai/keys"
+    );
+  }
+
+  // 1. Exact "provider/id"
+  const exact = models.find((m) => formatModelRef(m).toLowerCase() === query);
+  if (exact) return exact;
+
+  // 2. Exact id, ignoring the provider
+  const byId = models.find((m) => m.id.toLowerCase() === query);
+  if (byId) return byId;
+
+  // 3. Fuzzy
+  let best: ImageGenModel | undefined;
+  let bestScore = 0;
+
+  for (const model of models) {
+    const id = model.id.toLowerCase();
+    const full = formatModelRef(model).toLowerCase();
+    const name = (model.name ?? model.id).toLowerCase();
+
+    let score = 0;
+    if (id.includes(query) || full.includes(query)) {
+      score = 60 + (query.length / id.length) * 30;
+    } else if (name.includes(query)) {
+      score = 40 + (query.length / name.length) * 20;
+    }
+
+    if (score > bestScore) {
+      bestScore = score;
+      best = model;
+    }
+  }
+
+  if (best && bestScore > 0) return best;
+
+  const sample = models.slice(0, 10).map((m) => `  ${formatModelRef(m)}`).join("\n");
+  return (
+    `Unknown image model "${input}".\n` +
+    `Available models (${models.length} total):\n${sample}` +
+    (models.length > 10 ? "\n  …run /unipi:image-settings to browse all" : "")
+  );
+}
+
+/**
+ * List vision-capable chat models — those accepting image input.
+ *
+ * `Model.input` is `("text" | "image")[]` in pi-ai. Models that do not declare
+ * the field are excluded rather than assumed capable, so a bad guess never
+ * produces a confusing API error.
+ */
+export function listVisionModels(registry: ChatModelRegistry): VisionModel[] {
+  let models: unknown[];
+  try {
+    models = registry.getAvailable?.() ?? registry.getAll();
+  } catch {
+    return [];
+  }
+
+  return models.filter(isVisionModel);
+}
+
+function isVisionModel(model: unknown): model is VisionModel {
+  if (model === null || typeof model !== "object") return false;
+  const candidate = model as Partial<VisionModel>;
+  if (typeof candidate.id !== "string" || typeof candidate.provider !== "string") {
+    return false;
+  }
+  return Array.isArray(candidate.input) && candidate.input.includes("image");
+}
+
+/**
+ * Resolve a vision-model reference, restricted to image-capable models.
+ *
+ * Rejecting a text-only model here gives a much clearer message than letting
+ * the provider fail on an unexpected image part.
+ */
+export function resolveVisionModel(
+  input: string,
+  registry: ChatModelRegistry,
+): VisionModel | string {
+  const vision = listVisionModels(registry);
+
+  if (vision.length === 0) {
+    return (
+      "No vision-capable models are configured.\n" +
+      "→ image_recognize needs a model that accepts image input " +
+      "(e.g. anthropic/claude-sonnet, openai/gpt-5, google/gemini-3-pro).\n" +
+      "→ Configure one with /model or /unipi:image-settings."
+    );
+  }
+
+  const query = input.trim().toLowerCase();
+  if (!query) return "No model specified.";
+
+  const exact = vision.find((m) => formatModelRef(m).toLowerCase() === query);
+  if (exact) return exact;
+
+  const byId = vision.find((m) => m.id.toLowerCase() === query);
+  if (byId) return byId;
+
+  let best: VisionModel | undefined;
+  let bestScore = 0;
+
+  for (const model of vision) {
+    const id = model.id.toLowerCase();
+    const full = formatModelRef(model).toLowerCase();
+    const name = (model.name ?? model.id).toLowerCase();
+
+    let score = 0;
+    if (id.includes(query) || full.includes(query)) {
+      score = 60 + (query.length / id.length) * 30;
+    } else if (name.includes(query)) {
+      score = 40 + (query.length / name.length) * 20;
+    }
+
+    if (score > bestScore) {
+      bestScore = score;
+      best = model;
+    }
+  }
+
+  if (best && bestScore > 0) return best;
+
+  // A known model that simply cannot see gets a targeted message.
+  let all: unknown[] = [];
+  try {
+    all = registry.getAvailable?.() ?? registry.getAll();
+  } catch {
+    all = [];
+  }
+  const knownButBlind = all.some((m) => {
+    const candidate = m as Partial<VisionModel>;
+    if (typeof candidate.id !== "string" || typeof candidate.provider !== "string") {
+      return false;
+    }
+    return (
+      formatModelRef(candidate as VisionModel).toLowerCase() === query ||
+      candidate.id.toLowerCase() === query
+    );
+  });
+
+  if (knownButBlind) {
+    return `Model "${input}" does not accept image input. Vision-capable models: ${vision
+      .slice(0, 5)
+      .map(formatModelRef)
+      .join(", ")}`;
+  }
+
+  return (
+    `Unknown model "${input}".\n` +
+    `Vision-capable models: ${vision.map(formatModelRef).join(", ")}`
+  );
+}