@pi-unipi/image 2.2.0

package/src/index.ts

@@ -0,0 +1,99 @@
+/**
+ * @pi-unipi/image — Extension entry
+ *
+ * Provides the `image_generate` and `image_recognize` agent tools plus the
+ * `/unipi:image-settings` command.
+ */
+
+import { dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import {
+  IMAGE_COMMANDS,
+  IMAGE_TOOLS,
+  MODULES,
+  UNIPI_EVENTS,
+  UNIPI_PREFIX,
+  emitEvent,
+  getPackageVersion,
+} from "@pi-unipi/core";
+
+import { registerImageCommands } from "./commands.js";
+import { registerImageTools } from "./tools.js";
+import { listImageGenModels, listVisionModels, type ChatModelRegistry } from "./models.js";
+import { loadConfig } from "./settings.js";
+
+const VERSION = getPackageVersion(dirname(fileURLToPath(import.meta.url)));
+
+/** Info-screen registry, read off the global to avoid load-order coupling. */
+function getInfoRegistry() {
+  return (
+    globalThis as {
+      __unipi_info_registry?: {
+        registerGroup(group: unknown): void;
+      };
+    }
+  ).__unipi_info_registry;
+}
+
+export default function (pi: ExtensionAPI) {
+  registerImageTools(pi);
+  registerImageCommands(pi);
+
+  pi.on("session_start", async (_event, ctx) => {
+    const config = loadConfig();
+
+    const tools: string[] = [];
+    if (config.generate.enabled) tools.push(IMAGE_TOOLS.GENERATE);
+    if (config.recognize.enabled) tools.push(IMAGE_TOOLS.RECOGNIZE);
+
+    emitEvent(pi, UNIPI_EVENTS.MODULE_READY, {
+      name: MODULES.IMAGE,
+      version: VERSION,
+      commands: [`${UNIPI_PREFIX}${IMAGE_COMMANDS.SETTINGS}`],
+      tools,
+    });
+
+    const registry = getInfoRegistry();
+    if (!registry) return;
+
+    registry.registerGroup({
+      id: "image",
+      name: "Image",
+      icon: "🎨",
+      priority: 55,
+      config: {
+        showByDefault: true,
+        stats: [
+          { id: "generate", label: "Generate", show: true },
+          { id: "recognize", label: "Recognize", show: true },
+          { id: "visionModels", label: "Vision Models", show: true },
+        ],
+      },
+      dataProvider: async () => {
+        const current = loadConfig();
+
+        const genModels = await listImageGenModels();
+        const generate = current.generate.enabled
+          ? genModels.length > 0
+            ? current.generate.model
+            : "No image models available"
+          : "Disabled";
+
+        const chatRegistry = (ctx as unknown as { modelRegistry?: ChatModelRegistry })
+          .modelRegistry;
+        const vision = chatRegistry ? listVisionModels(chatRegistry) : [];
+
+        const recognize = current.recognize.enabled
+          ? current.recognize.model || "Session model"
+          : "Disabled";
+
+        return {
+          generate: { value: generate },
+          recognize: { value: recognize },
+          visionModels: { value: String(vision.length) },
+        };
+      },
+    });
+  });
+}

package/src/models.ts

@@ -0,0 +1,290 @@
+/**
+ * @pi-unipi/image — Model discovery and resolution
+ *
+ * Two different model families are involved:
+ *
+ * - **Image generation** uses pi-ai's `ImagesModel` catalog, which
+ *   pi-coding-agent does not expose on `ExtensionContext` — it is reached
+ *   through the pi-ai subpath exports.
+ * - **Image recognition** uses ordinary chat models filtered to those whose
+ *   `input` modality includes `"image"` (`ctx.modelRegistry`).
+ */
+
+/** A generation model, kept structural to avoid deep pi-ai type coupling. */
+export interface ImageGenModel {
+  id: string;
+  name?: string;
+  provider: string;
+  api: string;
+  baseUrl?: string;
+  input?: string[];
+  output?: string[];
+  [key: string]: unknown;
+}
+
+/** A vision-capable chat model. */
+export interface VisionModel {
+  id: string;
+  name?: string;
+  provider: string;
+  input?: string[];
+}
+
+/** Minimal chat-model registry surface (pi's ModelRegistry). */
+export interface ChatModelRegistry {
+  find(provider: string, modelId: string): unknown;
+  getAll(): unknown[];
+  getAvailable?(): unknown[];
+  getApiKeyForProvider?(provider: string): Promise<string | undefined>;
+}
+
+/** Split "provider/model-id" — the model id may itself contain slashes. */
+export function splitModelRef(ref: string): { provider: string; id: string } | null {
+  const trimmed = ref.trim();
+  const slash = trimmed.indexOf("/");
+  if (slash <= 0 || slash === trimmed.length - 1) return null;
+  return { provider: trimmed.slice(0, slash), id: trimmed.slice(slash + 1) };
+}
+
+/** Format a model as "provider/model-id". */
+export function formatModelRef(model: { provider: string; id: string }): string {
+  return `${model.provider}/${model.id}`;
+}
+
+/**
+ * pi-ai's runtime image-model collection: model catalog, auth resolution and
+ * generation in one object. Only the parts used here are typed.
+ */
+export interface ImagesModelsLike {
+  getModels(provider?: string): readonly ImageGenModel[];
+  getModel(provider: string, id: string): ImageGenModel | undefined;
+  getAuth(model: ImageGenModel): Promise<{ apiKey?: string } | undefined>;
+  generateImages(
+    model: ImageGenModel,
+    context: { input: Array<{ type: string; text?: string }> },
+    options?: { apiKey?: string; signal?: AbortSignal },
+  ): Promise<unknown>;
+}
+
+let cachedImagesModels: ImagesModelsLike | null = null;
+let imagesModelsAttempted = false;
+
+/**
+ * Load pi-ai's built-in images collection.
+ *
+ * `getImageModels`/`generateImages` are not re-exported from the pi-ai package
+ * root, but `providers/all` exports `builtinImagesModels()`, which is the
+ * supported entry point and also resolves auth. A failure is not fatal: it
+ * degrades to null and callers report that no models are available.
+ */
+export async function getImagesModels(): Promise<ImagesModelsLike | null> {
+  if (cachedImagesModels || imagesModelsAttempted) return cachedImagesModels;
+  imagesModelsAttempted = true;
+
+  try {
+    const mod = (await import("@earendil-works/pi-ai/providers/all")) as unknown as {
+      builtinImagesModels?: () => ImagesModelsLike;
+    };
+    if (typeof mod.builtinImagesModels === "function") {
+      cachedImagesModels = mod.builtinImagesModels();
+    }
+  } catch {
+    cachedImagesModels = null;
+  }
+
+  return cachedImagesModels;
+}
+
+/** List available image-generation models. Empty when unavailable. */
+export async function listImageGenModels(): Promise<ImageGenModel[]> {
+  const images = await getImagesModels();
+  if (!images) return [];
+  try {
+    return [...images.getModels()];
+  } catch {
+    return [];
+  }
+}
+
+/** Inject a stub images collection. Test-only. */
+export function __setImagesModelsForTests(models: ImagesModelsLike | null): void {
+  cachedImagesModels = models;
+  imagesModelsAttempted = models !== null;
+}
+
+/** Reset the model cache. Test-only. */
+export function __resetModelCacheForTests(): void {
+  cachedImagesModels = null;
+  imagesModelsAttempted = false;
+}
+
+/**
+ * Resolve a generation-model reference against the catalog.
+ *
+ * Exact "provider/id" first, then a scored fuzzy match so "flux" or
+ * "gemini-3-pro" work. Returns an error string (not a throw) so the tool can
+ * surface it as a normal tool error listing the alternatives.
+ */
+export function resolveImageGenModel(
+  input: string,
+  models: ImageGenModel[],
+): ImageGenModel | string {
+  const query = input.trim().toLowerCase();
+  if (!query) return "No image model specified.";
+  if (models.length === 0) {
+    return (
+      "No image generation models are available.\n" +
+      "→ Image generation requires an OpenRouter account: https://openrouter.ai/keys"
+    );
+  }
+
+  // 1. Exact "provider/id"
+  const exact = models.find((m) => formatModelRef(m).toLowerCase() === query);
+  if (exact) return exact;
+
+  // 2. Exact id, ignoring the provider
+  const byId = models.find((m) => m.id.toLowerCase() === query);
+  if (byId) return byId;
+
+  // 3. Fuzzy
+  let best: ImageGenModel | undefined;
+  let bestScore = 0;
+
+  for (const model of models) {
+    const id = model.id.toLowerCase();
+    const full = formatModelRef(model).toLowerCase();
+    const name = (model.name ?? model.id).toLowerCase();
+
+    let score = 0;
+    if (id.includes(query) || full.includes(query)) {
+      score = 60 + (query.length / id.length) * 30;
+    } else if (name.includes(query)) {
+      score = 40 + (query.length / name.length) * 20;
+    }
+
+    if (score > bestScore) {
+      bestScore = score;
+      best = model;
+    }
+  }
+
+  if (best && bestScore > 0) return best;
+
+  const sample = models.slice(0, 10).map((m) => `  ${formatModelRef(m)}`).join("\n");
+  return (
+    `Unknown image model "${input}".\n` +
+    `Available models (${models.length} total):\n${sample}` +
+    (models.length > 10 ? "\n  …run /unipi:image-settings to browse all" : "")
+  );
+}
+
+/**
+ * List vision-capable chat models — those accepting image input.
+ *
+ * `Model.input` is `("text" | "image")[]` in pi-ai. Models that do not declare
+ * the field are excluded rather than assumed capable, so a bad guess never
+ * produces a confusing API error.
+ */
+export function listVisionModels(registry: ChatModelRegistry): VisionModel[] {
+  let models: unknown[];
+  try {
+    models = registry.getAvailable?.() ?? registry.getAll();
+  } catch {
+    return [];
+  }
+
+  return models.filter(isVisionModel);
+}
+
+function isVisionModel(model: unknown): model is VisionModel {
+  if (model === null || typeof model !== "object") return false;
+  const candidate = model as Partial<VisionModel>;
+  if (typeof candidate.id !== "string" || typeof candidate.provider !== "string") {
+    return false;
+  }
+  return Array.isArray(candidate.input) && candidate.input.includes("image");
+}
+
+/**
+ * Resolve a vision-model reference, restricted to image-capable models.
+ *
+ * Rejecting a text-only model here gives a much clearer message than letting
+ * the provider fail on an unexpected image part.
+ */
+export function resolveVisionModel(
+  input: string,
+  registry: ChatModelRegistry,
+): VisionModel | string {
+  const vision = listVisionModels(registry);
+
+  if (vision.length === 0) {
+    return (
+      "No vision-capable models are configured.\n" +
+      "→ image_recognize needs a model that accepts image input " +
+      "(e.g. anthropic/claude-sonnet, openai/gpt-5, google/gemini-3-pro).\n" +
+      "→ Configure one with /model or /unipi:image-settings."
+    );
+  }
+
+  const query = input.trim().toLowerCase();
+  if (!query) return "No model specified.";
+
+  const exact = vision.find((m) => formatModelRef(m).toLowerCase() === query);
+  if (exact) return exact;
+
+  const byId = vision.find((m) => m.id.toLowerCase() === query);
+  if (byId) return byId;
+
+  let best: VisionModel | undefined;
+  let bestScore = 0;
+
+  for (const model of vision) {
+    const id = model.id.toLowerCase();
+    const full = formatModelRef(model).toLowerCase();
+    const name = (model.name ?? model.id).toLowerCase();
+
+    let score = 0;
+    if (id.includes(query) || full.includes(query)) {
+      score = 60 + (query.length / id.length) * 30;
+    } else if (name.includes(query)) {
+      score = 40 + (query.length / name.length) * 20;
+    }
+
+    if (score > bestScore) {
+      bestScore = score;
+      best = model;
+    }
+  }
+
+  if (best && bestScore > 0) return best;
+
+  // A known model that simply cannot see gets a targeted message.
+  let all: unknown[] = [];
+  try {
+    all = registry.getAvailable?.() ?? registry.getAll();
+  } catch {
+    all = [];
+  }
+  const knownButBlind = all.some((m) => {
+    const candidate = m as Partial<VisionModel>;
+    if (typeof candidate.id !== "string" || typeof candidate.provider !== "string") {
+      return false;
+    }
+    return (
+      formatModelRef(candidate as VisionModel).toLowerCase() === query ||
+      candidate.id.toLowerCase() === query
+    );
+  });
+
+  if (knownButBlind) {
+    return `Model "${input}" does not accept image input. Vision-capable models: ${vision
+      .slice(0, 5)
+      .map(formatModelRef)
+      .join(", ")}`;
+  }
+
+  return (
+    `Unknown model "${input}".\n` +
+    `Vision-capable models: ${vision.map(formatModelRef).join(", ")}`
+  );
+}

package/src/recognize.ts

@@ -0,0 +1,223 @@
+/**
+ * @pi-unipi/image — Image recognition
+ *
+ * Sends an image plus a question to a vision-capable chat model. Providers
+ * differ in how image parts are encoded, so the request is built per API
+ * family (mirroring `packages/notify/summarize.ts`).
+ */
+
+import type { LoadedImage } from "./image-source.js";
+
+/** How long to wait for a vision response. Images are slow. */
+const DEFAULT_TIMEOUT_MS = 120_000;
+
+/** Cap the reply so a verbose model cannot flood the context. */
+const DEFAULT_MAX_TOKENS = 2048;
+
+export interface RecognizeOptions {
+  image: LoadedImage;
+  /** The question to ask about the image. */
+  prompt: string;
+  /** System prompt steering the analysis. */
+  systemPrompt: string;
+  apiKey: string;
+  baseUrl: string;
+  /** pi-ai `Model.api`, e.g. "anthropic-messages". */
+  api: string;
+  modelId: string;
+  signal?: AbortSignal;
+  timeoutMs?: number;
+  maxTokens?: number;
+  /** Injectable fetch, for tests. */
+  fetchImpl?: typeof fetch;
+}
+
+export interface RecognizeResult {
+  text: string;
+  model: string;
+}
+
+/** Combine an external abort signal with an internal timeout. */
+function createSignal(
+  timeoutMs: number,
+  external?: AbortSignal,
+): { signal: AbortSignal; cleanup: () => void } {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+
+  const onAbort = () => controller.abort();
+  external?.addEventListener("abort", onAbort, { once: true });
+
+  return {
+    signal: controller.signal,
+    cleanup: () => {
+      clearTimeout(timer);
+      external?.removeEventListener("abort", onAbort);
+    },
+  };
+}
+
+/** Extract the useful part of a provider error body. */
+async function describeHttpError(response: Response): Promise<string> {
+  let detail = "";
+  try {
+    const body = await response.text();
+    try {
+      const parsed = JSON.parse(body) as { error?: { message?: string } | string };
+      detail =
+        typeof parsed.error === "string"
+          ? parsed.error
+          : parsed.error?.message ?? body.slice(0, 300);
+    } catch {
+      detail = body.slice(0, 300);
+    }
+  } catch {
+    // Body unavailable — status alone will have to do.
+  }
+
+  const base = `Vision request failed: ${response.status} ${response.statusText}`;
+  return detail ? `${base}\n${detail}` : base;
+}
+
+/** Anthropic Messages API — image parts use a nested `source` object. */
+async function callAnthropic(options: RecognizeOptions): Promise<string> {
+  const {
+    image, prompt, systemPrompt, apiKey, baseUrl, modelId,
+    timeoutMs = DEFAULT_TIMEOUT_MS, maxTokens = DEFAULT_MAX_TOKENS,
+    signal: external, fetchImpl = fetch,
+  } = options;
+
+  const { signal, cleanup } = createSignal(timeoutMs, external);
+
+  try {
+    const response = await fetchImpl(`${baseUrl.replace(/\/$/, "")}/messages`, {
+      method: "POST",
+      headers: {
+        "x-api-key": apiKey,
+        "anthropic-version": "2023-06-01",
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        model: modelId,
+        max_tokens: maxTokens,
+        system: systemPrompt,
+        messages: [
+          {
+            role: "user",
+            content: [
+              {
+                type: "image",
+                source: {
+                  type: "base64",
+                  media_type: image.mimeType,
+                  data: image.data,
+                },
+              },
+              { type: "text", text: prompt },
+            ],
+          },
+        ],
+      }),
+      signal,
+    });
+
+    if (!response.ok) throw new Error(await describeHttpError(response));
+
+    const data = (await response.json()) as {
+      content?: Array<{ type?: string; text?: string }>;
+    };
+
+    return (data.content ?? [])
+      .filter((block) => block.type === "text" && typeof block.text === "string")
+      .map((block) => block.text as string)
+      .join("\n")
+      .trim();
+  } finally {
+    cleanup();
+  }
+}
+
+/** OpenAI-compatible chat completions — image parts use a data: URL. */
+async function callOpenAICompatible(options: RecognizeOptions): Promise<string> {
+  const {
+    image, prompt, systemPrompt, apiKey, baseUrl, modelId,
+    timeoutMs = DEFAULT_TIMEOUT_MS, maxTokens = DEFAULT_MAX_TOKENS,
+    signal: external, fetchImpl = fetch,
+  } = options;
+
+  const { signal, cleanup } = createSignal(timeoutMs, external);
+
+  try {
+    const response = await fetchImpl(
+      `${baseUrl.replace(/\/$/, "")}/chat/completions`,
+      {
+        method: "POST",
+        headers: {
+          Authorization: `Bearer ${apiKey}`,
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+          model: modelId,
+          max_tokens: maxTokens,
+          messages: [
+            { role: "system", content: systemPrompt },
+            {
+              role: "user",
+              content: [
+                { type: "text", text: prompt },
+                {
+                  type: "image_url",
+                  image_url: {
+                    url: `data:${image.mimeType};base64,${image.data}`,
+                  },
+                },
+              ],
+            },
+          ],
+        }),
+        signal,
+      },
+    );
+
+    if (!response.ok) throw new Error(await describeHttpError(response));
+
+    const data = (await response.json()) as {
+      choices?: Array<{ message?: { content?: string | Array<{ text?: string }> } }>;
+    };
+
+    const content = data.choices?.[0]?.message?.content;
+    if (typeof content === "string") return content.trim();
+    if (Array.isArray(content)) {
+      return content.map((part) => part?.text ?? "").join("").trim();
+    }
+    return "";
+  } finally {
+    cleanup();
+  }
+}
+
+/**
+ * Analyze an image with a vision model.
+ * @throws {Error} with an actionable message on failure.
+ */
+export async function recognizeImage(
+  options: RecognizeOptions,
+): Promise<RecognizeResult> {
+  if (!options.apiKey) {
+    throw new Error(
+      "No API key available for the selected vision model.\n" +
+        "→ Sign in with /login, or set the provider's API key environment variable.",
+    );
+  }
+
+  const text =
+    options.api === "anthropic-messages"
+      ? await callAnthropic(options)
+      : await callOpenAICompatible(options);
+
+  if (!text) {
+    throw new Error("The model returned an empty response for this image.");
+  }
+
+  return { text, model: options.modelId };
+}