@pi-unipi/unipi 2.2.7 → 2.4.0

package/packages/image/src/openai-images-api.ts

@@ -0,0 +1,282 @@
+/**
+ * @pi-unipi/image — Generic OpenAI-compatible images adapter
+ *
+ * ONE adapter for every provider, rather than per-provider code. It speaks the
+ * OpenAI `POST {baseUrl}/images/generations` shape, which every gateway we have
+ * tested implements (OpenAI itself, OpenRouter, and OmniRoute's fan-out to
+ * openrouter/antigravity/codex/fal-ai backends).
+ *
+ * Why not pi-ai's built-in `api/openrouter-images`?
+ * Despite the name it drives `chat.completions` with `modalities:["image"]`.
+ * Gateways that do not implement that extension answer HTTP 200 with the model
+ * *narrating* the image ("Here's the image with the circle changed…") while
+ * silently dropping `message.images`. That is invisible data loss, so we use
+ * the dedicated images endpoint instead.
+ *
+ * Editing rides the same endpoint: `POST /images/generations` with an `image`
+ * array. `/images/edits` (multipart) is NOT used — gateways reject it for most
+ * providers ("Image edit is not supported for built-in provider ...").
+ */
+
+import type { ImageGenModel } from "./models.js";
+
+/** pi-ai's `ImagesContext` input parts. */
+export interface ImagesInputPart {
+  type: string;
+  text?: string;
+  data?: string;
+  mimeType?: string;
+}
+
+export interface ImagesContextLike {
+  input: ImagesInputPart[];
+}
+
+export interface ImagesOptionsLike {
+  apiKey?: string;
+  signal?: AbortSignal;
+  headers?: Record<string, string | null>;
+  timeoutMs?: number;
+  /** Injectable fetch, for tests. */
+  fetchImpl?: typeof fetch;
+}
+
+/** pi-ai's `AssistantImages`. */
+export interface AssistantImagesLike {
+  api: string;
+  provider: string;
+  model: string;
+  output: Array<{ type: string; text?: string; data?: string; mimeType?: string }>;
+  stopReason: "stop" | "error" | "aborted";
+  errorMessage?: string;
+  timestamp: number;
+}
+
+/** Images are slow — a minute is not unusual for a large model. */
+const DEFAULT_TIMEOUT_MS = 240_000;
+
+/**
+ * One returned image, normalized.
+ *
+ * Gateways disagree on the item shape; all three observed forms are accepted:
+ *   - `{ b64_json, media_type }`   — openrouter/* (note `media_type`, not `mimeType`)
+ *   - `{ b64_json, revised_prompt }` — antigravity/*
+ *   - `{ url: "data:image/png;base64,…" }` — codex/*
+ * A plain http(s) `url` is also tolerated and reported as text, since we cannot
+ * inline bytes we did not fetch.
+ */
+interface RawImageItem {
+  b64_json?: unknown;
+  url?: unknown;
+  media_type?: unknown;
+  mime_type?: unknown;
+  mimeType?: unknown;
+  revised_prompt?: unknown;
+}
+
+function asString(value: unknown): string | undefined {
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+
+/** Pull `{ data, mimeType }` out of one response item, whatever its shape. */
+export function normalizeImageItem(
+  item: RawImageItem,
+): { data: string; mimeType: string } | { text: string } | null {
+  const declared =
+    asString(item.media_type) ?? asString(item.mime_type) ?? asString(item.mimeType);
+
+  const b64 = asString(item.b64_json);
+  if (b64) return { data: b64, mimeType: declared ?? "image/png" };
+
+  const url = asString(item.url);
+  if (!url) return null;
+
+  // codex/* returns the bytes as a data: URL rather than b64_json.
+  const dataUrl = /^data:([^;,]+)(?:;[^,]*)*,(.*)$/s.exec(url);
+  if (dataUrl) {
+    const [, mime, payload] = dataUrl;
+    if (payload) return { data: payload, mimeType: declared ?? mime ?? "image/png" };
+    return null;
+  }
+
+  // A remote URL: surface it rather than silently dropping the result.
+  return { text: `Image available at: ${url}` };
+}
+
+/** Strip a trailing slash so `${base}/images/generations` is well-formed. */
+function joinUrl(baseUrl: string, suffix: string): string {
+  return `${baseUrl.replace(/\/+$/, "")}/${suffix.replace(/^\/+/, "")}`;
+}
+
+/**
+ * Model ids are sent to the gateway verbatim.
+ *
+ * Do NOT try to "repair" a doubled-looking segment. OmniRoute genuinely serves
+ * `fal-ai/fal-ai/nano-banana-pro` (provider `fal-ai` + model `fal-ai/nano-...`),
+ * and rewriting it to `fal-ai/nano-banana-pro` yields a 404. Confusingly the
+ * gateway *also* advertises a `fal/...` alias in `/v1/models` that the images
+ * endpoint then rejects with "Invalid image model" — an upstream inconsistency
+ * we surface rather than guess around, because a wrong guess turns a clear
+ * error into a silently different model.
+ */
+export function normalizeModelId(id: string): string {
+  return id;
+}
+
+/** Merge caller headers, dropping keys explicitly suppressed with null. */
+function buildHeaders(
+  apiKey: string,
+  extra?: Record<string, string | null>,
+): Record<string, string> {
+  const headers: Record<string, string> = {
+    Authorization: `Bearer ${apiKey}`,
+    "Content-Type": "application/json",
+  };
+  for (const [key, value] of Object.entries(extra ?? {})) {
+    if (value === null) delete headers[key];
+    else headers[key] = value;
+  }
+  return headers;
+}
+
+/** Best-effort extraction of a provider error message. */
+function describeError(status: number, statusText: string, body: string): string {
+  let detail = body.slice(0, 300);
+  try {
+    const parsed = JSON.parse(body) as { error?: { message?: string } | string };
+    if (typeof parsed.error === "string") detail = parsed.error;
+    else if (parsed.error?.message) detail = parsed.error.message;
+  } catch {
+    // Non-JSON body — the truncated text is the best we have.
+  }
+  return `${status} ${statusText}${detail ? `: ${detail}` : ""}`;
+}
+
+/** Add guidance for the gateway's confusing model-id errors. */
+function annotateModelError(message: string, model: ImageGenModel): string {
+  if (/invalid image model|not found|unknown model/i.test(message)) {
+    return (
+      `${message}\n` +
+      `→ Model id sent: "${model.id}" (provider "${model.provider}").\n` +
+      "→ Some gateways list aliases they cannot serve. Try the id exactly as it " +
+      "appears in the provider's own catalog, or pick another with /unipi:image-settings."
+    );
+  }
+  return message;
+}
+
+/**
+ * Generate (or edit) images against an OpenAI-compatible endpoint.
+ *
+ * Satisfies pi-ai's `ProviderImages` interface, so the result is returned —
+ * never thrown — with `stopReason: "error"` on failure.
+ */
+export async function generateImages(
+  model: ImageGenModel,
+  context: ImagesContextLike,
+  options?: ImagesOptionsLike,
+): Promise<AssistantImagesLike> {
+  const result: AssistantImagesLike = {
+    api: model.api || "openai-images",
+    provider: model.provider,
+    model: model.id,
+    output: [],
+    stopReason: "stop",
+    timestamp: Date.now(),
+  };
+
+  const fetchImpl = options?.fetchImpl ?? fetch;
+  const controller = new AbortController();
+  const timer = setTimeout(
+    () => controller.abort(),
+    options?.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+  );
+  const onAbort = () => controller.abort();
+  options?.signal?.addEventListener("abort", onAbort, { once: true });
+
+  try {
+    const apiKey = options?.apiKey;
+    if (!apiKey) throw new Error(`No API key for provider: ${model.provider}`);
+    if (!model.baseUrl) {
+      throw new Error(`No baseUrl for image model ${model.provider}/${model.id}`);
+    }
+
+    // Text parts form the prompt; image parts switch the request into edit mode.
+    const prompt = context.input
+      .filter((part) => part.type === "text" && part.text)
+      .map((part) => part.text as string)
+      .join("\n")
+      .trim();
+
+    const images = context.input
+      .filter((part) => part.type === "image" && part.data)
+      .map((part) => `data:${part.mimeType || "image/png"};base64,${part.data}`);
+
+    if (!prompt) throw new Error("A non-empty prompt is required.");
+
+    const body: Record<string, unknown> = {
+      model: normalizeModelId(model.id),
+      prompt,
+    };
+    // Only send `image` for edits; some backends reject an empty array.
+    if (images.length > 0) body.image = images;
+
+    const response = await fetchImpl(joinUrl(model.baseUrl, "images/generations"), {
+      method: "POST",
+      headers: buildHeaders(apiKey, options?.headers),
+      body: JSON.stringify(body),
+      signal: controller.signal,
+    });
+
+    if (!response.ok) {
+      throw new Error(
+        describeError(response.status, response.statusText, await response.text()),
+      );
+    }
+
+    const payload = (await response.json()) as {
+      data?: RawImageItem[];
+      error?: { message?: string };
+    };
+
+    if (payload.error?.message) throw new Error(payload.error.message);
+
+    for (const item of payload.data ?? []) {
+      const normalized = normalizeImageItem(item);
+      if (!normalized) continue;
+      if ("text" in normalized) {
+        result.output.push({ type: "text", text: normalized.text });
+      } else {
+        result.output.push({
+          type: "image",
+          data: normalized.data,
+          mimeType: normalized.mimeType,
+        });
+      }
+      const revised = asString(item.revised_prompt);
+      if (revised && revised !== prompt) {
+        result.output.push({ type: "text", text: `Revised prompt: ${revised}` });
+      }
+    }
+
+    if (result.output.every((part) => part.type !== "image")) {
+      throw new Error("The provider returned no image data.");
+    }
+
+    return result;
+  } catch (error) {
+    const aborted = options?.signal?.aborted || controller.signal.aborted;
+    result.stopReason = options?.signal?.aborted ? "aborted" : "error";
+    result.errorMessage =
+      aborted && !options?.signal?.aborted
+        ? "Image request timed out."
+        : annotateModelError(
+            error instanceof Error ? error.message : String(error),
+            model,
+          );
+    return result;
+  } finally {
+    clearTimeout(timer);
+    options?.signal?.removeEventListener("abort", onAbort);
+  }
+}

package/packages/image/src/register-providers.ts

@@ -0,0 +1,220 @@
+/**
+ * @pi-unipi/image — Bridge pi's chat providers into pi-ai's images collection
+ *
+ * pi-ai ships exactly one image provider (openrouter), so out of the box image
+ * generation demands an OpenRouter account even when the user has half a dozen
+ * other providers configured. pi's own registry knows those providers and their
+ * credentials, so we re-register each one as an *images* provider backed by the
+ * single generic OpenAI-compatible adapter.
+ *
+ * The result: any OpenAI-compatible provider the user configures in pi can
+ * generate and edit images with no image-specific setup, and no per-provider
+ * code here.
+ *
+ * ## Why capability detection stays heuristic
+ * pi's model registry cannot tell us which models emit images.
+ * `provider-composer.ts` builds each registered model from an explicit field
+ * list — `{id, name, api, provider, baseUrl, reasoning, input, cost,
+ * contextWindow, maxTokens, headers, compat}` — so an extension that attaches
+ * `output: ["image"]` has it silently dropped. `ProviderModelConfig` has no
+ * `output` field at all. Hence `looksLikeImageGenerator()` name-matching, plus
+ * explicit "provider/model-id" entry as the always-available escape hatch.
+ */
+
+import * as imagesApi from "./openai-images-api.js";
+import {
+  getImagesModels,
+  listRegistryImageGenModels,
+  type ChatModelRegistry,
+  type ImageGenModel,
+} from "./models.js";
+
+/** pi-ai's `createImagesProvider`, kept structural to avoid type coupling. */
+interface CreateImagesProviderFn {
+  (input: {
+    id: string;
+    name?: string;
+    auth: unknown;
+    models: readonly ImageGenModel[];
+    api: unknown;
+  }): unknown;
+}
+
+/** The subset of a registry provider we need. */
+interface ProviderLike {
+  id: string;
+  name?: string;
+  baseUrl?: string;
+}
+
+let registered = false;
+
+/** Reset registration state. Test-only. */
+export function __resetRegistrationForTests(): void {
+  registered = false;
+}
+
+/**
+ * Group discovered generator models by provider, attaching the provider's
+ * baseUrl so the adapter knows where to POST.
+ */
+export function groupModelsByProvider(
+  models: ImageGenModel[],
+  baseUrlFor: (provider: string) => string | undefined,
+): Map<string, { baseUrl: string; models: ImageGenModel[] }> {
+  const grouped = new Map<string, { baseUrl: string; models: ImageGenModel[] }>();
+
+  for (const model of models) {
+    const baseUrl = model.baseUrl ?? baseUrlFor(model.provider);
+    // Without an endpoint the adapter cannot issue a request; skip rather than
+    // register a provider that is guaranteed to fail.
+    if (!baseUrl) continue;
+
+    let entry = grouped.get(model.provider);
+    if (!entry) {
+      entry = { baseUrl, models: [] };
+      grouped.set(model.provider, entry);
+    }
+    entry.models.push({ ...model, baseUrl });
+  }
+
+  return grouped;
+}
+
+/** Read a provider's baseUrl out of pi's registry. */
+function providerBaseUrlLookup(
+  registry: ChatModelRegistry,
+): (provider: string) => string | undefined {
+  const cache = new Map<string, string | undefined>();
+
+  return (provider: string) => {
+    if (cache.has(provider)) return cache.get(provider);
+
+    let baseUrl: string | undefined;
+    try {
+      const models = (registry.getAvailable?.() ?? registry.getAll()) as Array<{
+        provider?: string;
+        baseUrl?: string;
+      }>;
+      baseUrl = models.find((m) => m?.provider === provider && m.baseUrl)?.baseUrl;
+    } catch {
+      baseUrl = undefined;
+    }
+
+    cache.set(provider, baseUrl);
+    return baseUrl;
+  };
+}
+
+/**
+ * Register every pi provider that looks capable of image generation into
+ * pi-ai's images collection.
+ *
+ * Idempotent and best-effort: a failure here must never break the extension,
+ * since generation still works for pi-ai's built-in providers.
+ *
+ * @returns the provider ids registered.
+ */
+export async function registerRegistryImageProviders(
+  registry: ChatModelRegistry | undefined,
+  options?: { force?: boolean },
+): Promise<string[]> {
+  if (!registry) return [];
+  if (registered && !options?.force) return [];
+
+  const images = await getImagesModels();
+  if (!images) return [];
+
+  // `setProvider` is on MutableImagesModels; the built-in collection provides
+  // it, but guard in case a future pi-ai hands back an immutable one.
+  const mutable = images as unknown as {
+    setProvider?: (provider: unknown) => void;
+    getProvider?: (id: string) => unknown;
+  };
+  if (typeof mutable.setProvider !== "function") return [];
+
+  let createImagesProvider: CreateImagesProviderFn;
+  try {
+    const mod = (await import("@earendil-works/pi-ai")) as unknown as {
+      createImagesProvider?: CreateImagesProviderFn;
+    };
+    if (typeof mod.createImagesProvider !== "function") return [];
+    createImagesProvider = mod.createImagesProvider;
+  } catch {
+    return [];
+  }
+
+  const discovered = listRegistryImageGenModels(registry);
+  if (discovered.length === 0) {
+    registered = true;
+    return [];
+  }
+
+  const grouped = groupModelsByProvider(discovered, providerBaseUrlLookup(registry));
+  const added: string[] = [];
+
+  for (const [providerId, { models }] of grouped) {
+    // Never shadow a provider pi-ai serves natively — its own implementation
+    // is better informed than our generic adapter.
+    try {
+      if (mutable.getProvider?.(providerId)) continue;
+    } catch {
+      // Treat a lookup failure as "not present" and attempt registration.
+    }
+
+    try {
+      const provider = createImagesProvider({
+        id: providerId,
+        name: providerId,
+        models,
+        api: imagesApi,
+        auth: {
+          apiKey: {
+            name: `${providerId} API key`,
+            // Resolve through pi's own auth storage so the user never logs in
+            // twice. `resolve` MUST return an AuthResult (`{ auth: {...} }`);
+            // returning a bare key fails silently at request time.
+            resolve: async () => {
+              const key = await resolveProviderKey(registry, providerId);
+              return key ? { auth: { apiKey: key }, source: `pi:${providerId}` } : undefined;
+            },
+          },
+        },
+      });
+
+      mutable.setProvider(provider);
+      added.push(providerId);
+    } catch {
+      // One bad provider must not stop the rest.
+    }
+  }
+
+  registered = true;
+  return added;
+}
+
+/** Resolve a provider key from pi's auth storage, falling back to the env. */
+async function resolveProviderKey(
+  registry: ChatModelRegistry,
+  provider: string,
+): Promise<string | undefined> {
+  try {
+    const key = await registry.getApiKeyForProvider?.(provider);
+    if (key) return key;
+  } catch {
+    // Fall through to the environment.
+  }
+  const envName = `${provider.toUpperCase().replace(/[^A-Z0-9]/g, "_")}_API_KEY`;
+  return process.env[envName] || undefined;
+}
+
+/** Provider ids pi-ai can currently generate with, after registration. */
+export function registeredProviderIds(images: {
+  getProviders?: () => ReadonlyArray<{ id: string }>;
+}): string[] {
+  try {
+    return (images.getProviders?.() ?? []).map((p) => p.id);
+  } catch {
+    return [];
+  }
+}

package/packages/image/src/tools.ts

@@ -12,7 +12,9 @@ import { IMAGE_TOOLS } from "@pi-unipi/core";
 
 import { generateImage } from "./generate.js";
 import { loadImage } from "./image-source.js";
+import { registerRegistryImageProviders } from "./register-providers.js";
 import {
+  findProviderBaseUrl,
   formatModelRef,
   listAllImageGenModels,
   resolveImageGenModel,
@@ -84,12 +86,15 @@ function registerGenerateTool(pi: ExtensionAPI): void {
     name: IMAGE_TOOLS.GENERATE,
     label: "Generate Image",
     description:
-      "Generate an image from a text prompt using an image model. " +
-      "The image is returned inline and, when enabled, saved to disk.",
+      "Generate an image from a text prompt, or edit an existing image by " +
+      "passing `image`. The result is returned inline and, when enabled, saved to disk.",
     promptSnippet: "Generate an image from a text prompt.",
     promptGuidelines: [
       "Use image_generate to create images from a text description.",
       "Write a detailed prompt — subject, style, composition and lighting all help.",
+      "Pass `image` to edit an existing image instead of generating a new one.",
+      "Editing regenerates the whole image, so unmentioned details may change.",
+      "Describe what you DO want; negation is unreliable in image models.",
       "Omit model to use the one configured in /unipi:image-settings.",
       "Generated images cost money per call; do not regenerate without being asked.",
     ],
@@ -97,6 +102,13 @@ function registerGenerateTool(pi: ExtensionAPI): void {
       prompt: Type.String({
         description: "Description of the image to generate. Be specific.",
       }),
+      image: Type.Optional(
+        Type.String({
+          description:
+            "Source image to edit: a local file path, data: URL, or base64 data. " +
+            "When set, the model edits this image instead of generating from scratch.",
+        }),
+      ),
       model: Type.Optional(
         Type.String({
           description:
@@ -109,22 +121,41 @@ function registerGenerateTool(pi: ExtensionAPI): void {
       try {
         const config = loadConfig();
         const registry = getRegistry(ctx);
+        // Bridge pi's own providers into pi-ai's images collection so the user
+        // is not forced onto OpenRouter. Idempotent and best-effort.
+        await registerRegistryImageProviders(registry);
         // Include image models contributed by registered providers, so the
         // tool can resolve anything the settings picker offers.
         const models = await listAllImageGenModels(registry);
 
         const requested = params.model?.trim() || config.generate.model;
-        const resolved = resolveImageGenModel(requested, models);
-        if (typeof resolved === "string") return errorResult(resolved);
+        const maybeResolved = resolveImageGenModel(requested, models);
+        if (typeof maybeResolved === "string") return errorResult(maybeResolved);
+
+        // A model may arrive without an endpoint — notably a user-typed
+        // "provider/model-id", accepted at face value. Fill it in from the
+        // registry so the adapter knows where to POST.
+        const registryBaseUrl = maybeResolved.baseUrl
+          ? undefined
+          : findProviderBaseUrl(registry, maybeResolved.provider);
+        const resolved = registryBaseUrl
+          ? { ...maybeResolved, baseUrl: registryBaseUrl }
+          : maybeResolved;
 
         // pi-ai resolves image auth from its own credential store; only fall
         // back to pi's chat-provider key when that comes up empty.
         const fallbackKey = await resolveApiKey(registry, resolved.provider);
 
+        // An input image switches the request into edit mode.
+        const sourceImage = params.image?.trim()
+          ? loadImage(params.image, ctx.cwd ?? process.cwd())
+          : undefined;
+
         const result = await generateImage({
           prompt: params.prompt,
           model: resolved,
           ...(fallbackKey ? { apiKey: fallbackKey } : {}),
+          ...(sourceImage ? { inputImage: sourceImage } : {}),
           signal,
           outputDir: config.generate.saveToDisk ? getOutputDir(config) : undefined,
         });
@@ -134,8 +165,8 @@ function registerGenerateTool(pi: ExtensionAPI): void {
           .filter((path): path is string => Boolean(path));
 
         const summary = [
-          `Generated ${result.images.length} image${result.images.length === 1 ? "" : "s"} ` +
-            `with ${formatModelRef(resolved)}.`,
+          `${sourceImage ? "Edited" : "Generated"} ${result.images.length} ` +
+            `image${result.images.length === 1 ? "" : "s"} with ${formatModelRef(resolved)}.`,
           saved.length > 0 ? `Saved to:\n${saved.map((p) => `  ${p}`).join("\n")}` : "",
           config.generate.saveToDisk && saved.length === 0
             ? "Could not write to the output directory — returning the image inline only."

package/packages/image/src/tui/settings-dialog.ts

@@ -23,6 +23,7 @@ import {
   type ChatModelRegistry,
 } from "../models.js";
 import { ImageModelSelectorOverlay, type SelectableModel } from "./model-selector.js";
+import { registerRegistryImageProviders } from "../register-providers.js";
 
 const EXIT = "__exit__";
 
@@ -239,6 +240,10 @@ async function collectModels(
     .modelRegistry;
 
   if (kind === "generate") {
+    // Bridge pi's providers in first, so a model the user can actually run is
+    // not flagged "no image route" purely because we had not registered it yet.
+    await registerRegistryImageProviders(registry);
+
     // Include models from providers registered by other extensions, not just
     // pi-ai's built-in OpenRouter catalog.
     const models = await listAllImageGenModels(registry);
@@ -253,7 +258,7 @@ async function collectModels(
       // so flag them rather than letting the user pick a dead option.
       unavailable:
         generating.length > 0 && !generating.includes(m.provider)
-          ? "cannot generate"
+          ? "no image route"
           : undefined,
     }));
   }

package/packages/info-screen/README.md

@@ -76,8 +76,8 @@ Settings in pi `settings.json`:
 {
   "unipi": {
     "infoScreen": {
-      "showOnBoot": true,
-      "bootTimeoutMs": 8000,
+      "bootMode": "auto-close",
+      "bootTimeoutMs": 2000,
       "groups": {
         "modules": { "show": true },
         "ralph": { "show": true },
@@ -91,8 +91,8 @@ Settings in pi `settings.json`:
 
 | Setting | Default | What It Does |
 |---------|---------|--------------|
-| `showOnBoot` | true | Show dashboard when session starts |
-| `bootTimeoutMs` | 8000 | How long to wait for modules before showing |
+| `bootMode` | `"auto-close"` | `"on"` keeps the dashboard up until dismissed, `"auto-close"` closes it after `bootTimeoutMs`, `"off"` never shows it |
+| `bootTimeoutMs` | 2000 | Auto-close delay, in ms. Any keypress cancels it. Ignored unless `bootMode` is `"auto-close"` |
 | `groups.{id}.show` | true | Toggle group visibility |
 | `groupOrder` | priority sort | Custom group ordering |