@pi-unipi/unipi 3.0.0-alpha.2 → 3.0.0-alpha.21

package/packages/image/src/openai-images-api.ts

@@ -1,282 +0,0 @@
-/**
- * @pi-unipi/image — Generic OpenAI-compatible images adapter
- *
- * ONE adapter for every provider, rather than per-provider code. It speaks the
- * OpenAI `POST {baseUrl}/images/generations` shape, which every gateway we have
- * tested implements (OpenAI itself, OpenRouter, and OmniRoute's fan-out to
- * openrouter/antigravity/codex/fal-ai backends).
- *
- * Why not pi-ai's built-in `api/openrouter-images`?
- * Despite the name it drives `chat.completions` with `modalities:["image"]`.
- * Gateways that do not implement that extension answer HTTP 200 with the model
- * *narrating* the image ("Here's the image with the circle changed…") while
- * silently dropping `message.images`. That is invisible data loss, so we use
- * the dedicated images endpoint instead.
- *
- * Editing rides the same endpoint: `POST /images/generations` with an `image`
- * array. `/images/edits` (multipart) is NOT used — gateways reject it for most
- * providers ("Image edit is not supported for built-in provider ...").
- */
-
-import type { ImageGenModel } from "./models.js";
-
-/** pi-ai's `ImagesContext` input parts. */
-export interface ImagesInputPart {
-  type: string;
-  text?: string;
-  data?: string;
-  mimeType?: string;
-}
-
-export interface ImagesContextLike {
-  input: ImagesInputPart[];
-}
-
-export interface ImagesOptionsLike {
-  apiKey?: string;
-  signal?: AbortSignal;
-  headers?: Record<string, string | null>;
-  timeoutMs?: number;
-  /** Injectable fetch, for tests. */
-  fetchImpl?: typeof fetch;
-}
-
-/** pi-ai's `AssistantImages`. */
-export interface AssistantImagesLike {
-  api: string;
-  provider: string;
-  model: string;
-  output: Array<{ type: string; text?: string; data?: string; mimeType?: string }>;
-  stopReason: "stop" | "error" | "aborted";
-  errorMessage?: string;
-  timestamp: number;
-}
-
-/** Images are slow — a minute is not unusual for a large model. */
-const DEFAULT_TIMEOUT_MS = 240_000;
-
-/**
- * One returned image, normalized.
- *
- * Gateways disagree on the item shape; all three observed forms are accepted:
- *   - `{ b64_json, media_type }`   — openrouter/* (note `media_type`, not `mimeType`)
- *   - `{ b64_json, revised_prompt }` — antigravity/*
- *   - `{ url: "data:image/png;base64,…" }` — codex/*
- * A plain http(s) `url` is also tolerated and reported as text, since we cannot
- * inline bytes we did not fetch.
- */
-interface RawImageItem {
-  b64_json?: unknown;
-  url?: unknown;
-  media_type?: unknown;
-  mime_type?: unknown;
-  mimeType?: unknown;
-  revised_prompt?: unknown;
-}
-
-function asString(value: unknown): string | undefined {
-  return typeof value === "string" && value.length > 0 ? value : undefined;
-}
-
-/** Pull `{ data, mimeType }` out of one response item, whatever its shape. */
-export function normalizeImageItem(
-  item: RawImageItem,
-): { data: string; mimeType: string } | { text: string } | null {
-  const declared =
-    asString(item.media_type) ?? asString(item.mime_type) ?? asString(item.mimeType);
-
-  const b64 = asString(item.b64_json);
-  if (b64) return { data: b64, mimeType: declared ?? "image/png" };
-
-  const url = asString(item.url);
-  if (!url) return null;
-
-  // codex/* returns the bytes as a data: URL rather than b64_json.
-  const dataUrl = /^data:([^;,]+)(?:;[^,]*)*,(.*)$/s.exec(url);
-  if (dataUrl) {
-    const [, mime, payload] = dataUrl;
-    if (payload) return { data: payload, mimeType: declared ?? mime ?? "image/png" };
-    return null;
-  }
-
-  // A remote URL: surface it rather than silently dropping the result.
-  return { text: `Image available at: ${url}` };
-}
-
-/** Strip a trailing slash so `${base}/images/generations` is well-formed. */
-function joinUrl(baseUrl: string, suffix: string): string {
-  return `${baseUrl.replace(/\/+$/, "")}/${suffix.replace(/^\/+/, "")}`;
-}
-
-/**
- * Model ids are sent to the gateway verbatim.
- *
- * Do NOT try to "repair" a doubled-looking segment. OmniRoute genuinely serves
- * `fal-ai/fal-ai/nano-banana-pro` (provider `fal-ai` + model `fal-ai/nano-...`),
- * and rewriting it to `fal-ai/nano-banana-pro` yields a 404. Confusingly the
- * gateway *also* advertises a `fal/...` alias in `/v1/models` that the images
- * endpoint then rejects with "Invalid image model" — an upstream inconsistency
- * we surface rather than guess around, because a wrong guess turns a clear
- * error into a silently different model.
- */
-export function normalizeModelId(id: string): string {
-  return id;
-}
-
-/** Merge caller headers, dropping keys explicitly suppressed with null. */
-function buildHeaders(
-  apiKey: string,
-  extra?: Record<string, string | null>,
-): Record<string, string> {
-  const headers: Record<string, string> = {
-    Authorization: `Bearer ${apiKey}`,
-    "Content-Type": "application/json",
-  };
-  for (const [key, value] of Object.entries(extra ?? {})) {
-    if (value === null) delete headers[key];
-    else headers[key] = value;
-  }
-  return headers;
-}
-
-/** Best-effort extraction of a provider error message. */
-function describeError(status: number, statusText: string, body: string): string {
-  let detail = body.slice(0, 300);
-  try {
-    const parsed = JSON.parse(body) as { error?: { message?: string } | string };
-    if (typeof parsed.error === "string") detail = parsed.error;
-    else if (parsed.error?.message) detail = parsed.error.message;
-  } catch {
-    // Non-JSON body — the truncated text is the best we have.
-  }
-  return `${status} ${statusText}${detail ? `: ${detail}` : ""}`;
-}
-
-/** Add guidance for the gateway's confusing model-id errors. */
-function annotateModelError(message: string, model: ImageGenModel): string {
-  if (/invalid image model|not found|unknown model/i.test(message)) {
-    return (
-      `${message}\n` +
-      `→ Model id sent: "${model.id}" (provider "${model.provider}").\n` +
-      "→ Some gateways list aliases they cannot serve. Try the id exactly as it " +
-      "appears in the provider's own catalog, or pick another with /unipi:settings."
-    );
-  }
-  return message;
-}
-
-/**
- * Generate (or edit) images against an OpenAI-compatible endpoint.
- *
- * Satisfies pi-ai's `ProviderImages` interface, so the result is returned —
- * never thrown — with `stopReason: "error"` on failure.
- */
-export async function generateImages(
-  model: ImageGenModel,
-  context: ImagesContextLike,
-  options?: ImagesOptionsLike,
-): Promise<AssistantImagesLike> {
-  const result: AssistantImagesLike = {
-    api: model.api || "openai-images",
-    provider: model.provider,
-    model: model.id,
-    output: [],
-    stopReason: "stop",
-    timestamp: Date.now(),
-  };
-
-  const fetchImpl = options?.fetchImpl ?? fetch;
-  const controller = new AbortController();
-  const timer = setTimeout(
-    () => controller.abort(),
-    options?.timeoutMs ?? DEFAULT_TIMEOUT_MS,
-  );
-  const onAbort = () => controller.abort();
-  options?.signal?.addEventListener("abort", onAbort, { once: true });
-
-  try {
-    const apiKey = options?.apiKey;
-    if (!apiKey) throw new Error(`No API key for provider: ${model.provider}`);
-    if (!model.baseUrl) {
-      throw new Error(`No baseUrl for image model ${model.provider}/${model.id}`);
-    }
-
-    // Text parts form the prompt; image parts switch the request into edit mode.
-    const prompt = context.input
-      .filter((part) => part.type === "text" && part.text)
-      .map((part) => part.text as string)
-      .join("\n")
-      .trim();
-
-    const images = context.input
-      .filter((part) => part.type === "image" && part.data)
-      .map((part) => `data:${part.mimeType || "image/png"};base64,${part.data}`);
-
-    if (!prompt) throw new Error("A non-empty prompt is required.");
-
-    const body: Record<string, unknown> = {
-      model: normalizeModelId(model.id),
-      prompt,
-    };
-    // Only send `image` for edits; some backends reject an empty array.
-    if (images.length > 0) body.image = images;
-
-    const response = await fetchImpl(joinUrl(model.baseUrl, "images/generations"), {
-      method: "POST",
-      headers: buildHeaders(apiKey, options?.headers),
-      body: JSON.stringify(body),
-      signal: controller.signal,
-    });
-
-    if (!response.ok) {
-      throw new Error(
-        describeError(response.status, response.statusText, await response.text()),
-      );
-    }
-
-    const payload = (await response.json()) as {
-      data?: RawImageItem[];
-      error?: { message?: string };
-    };
-
-    if (payload.error?.message) throw new Error(payload.error.message);
-
-    for (const item of payload.data ?? []) {
-      const normalized = normalizeImageItem(item);
-      if (!normalized) continue;
-      if ("text" in normalized) {
-        result.output.push({ type: "text", text: normalized.text });
-      } else {
-        result.output.push({
-          type: "image",
-          data: normalized.data,
-          mimeType: normalized.mimeType,
-        });
-      }
-      const revised = asString(item.revised_prompt);
-      if (revised && revised !== prompt) {
-        result.output.push({ type: "text", text: `Revised prompt: ${revised}` });
-      }
-    }
-
-    if (result.output.every((part) => part.type !== "image")) {
-      throw new Error("The provider returned no image data.");
-    }
-
-    return result;
-  } catch (error) {
-    const aborted = options?.signal?.aborted || controller.signal.aborted;
-    result.stopReason = options?.signal?.aborted ? "aborted" : "error";
-    result.errorMessage =
-      aborted && !options?.signal?.aborted
-        ? "Image request timed out."
-        : annotateModelError(
-            error instanceof Error ? error.message : String(error),
-            model,
-          );
-    return result;
-  } finally {
-    clearTimeout(timer);
-    options?.signal?.removeEventListener("abort", onAbort);
-  }
-}

package/packages/image/src/register-providers.ts

@@ -1,209 +0,0 @@
-/**
- * @pi-unipi/image — Bridge pi's chat providers into pi-ai's images collection
- *
- * pi-ai ships exactly one image provider (openrouter), so out of the box image
- * generation demands an OpenRouter account even when the user has half a dozen
- * other providers configured. pi's own registry knows those providers and their
- * credentials, so we re-register each one as an *images* provider backed by the
- * single generic OpenAI-compatible adapter.
- *
- * The result: any OpenAI-compatible provider the user configures in pi can
- * generate and edit images with no image-specific setup, and no per-provider
- * code here.
- *
- * ## Why capability detection stays heuristic
- * pi's model registry cannot tell us which models emit images.
- * `provider-composer.ts` builds each registered model from an explicit field
- * list — `{id, name, api, provider, baseUrl, reasoning, input, cost,
- * contextWindow, maxTokens, headers, compat}` — so an extension that attaches
- * `output: ["image"]` has it silently dropped. `ProviderModelConfig` has no
- * `output` field at all. Hence `looksLikeImageGenerator()` name-matching, plus
- * explicit "provider/model-id" entry as the always-available escape hatch.
- */
-
-import * as imagesApi from "./openai-images-api.js";
-import {
-  getImagesModels,
-  listRegistryImageGenModels,
-  type ChatModelRegistry,
-  type ImageGenModel,
-} from "./models.js";
-
-/** pi-ai's `createImagesProvider`, kept structural to avoid type coupling. */
-interface CreateImagesProviderFn {
-  (input: {
-    id: string;
-    name?: string;
-    auth: unknown;
-    models: readonly ImageGenModel[];
-    api: unknown;
-  }): unknown;
-}
-
-/** The subset of a registry provider we need. */
-interface ProviderLike {
-  id: string;
-  name?: string;
-  baseUrl?: string;
-}
-
-let registered = false;
-
-/** Reset registration state. Test-only. */
-export function __resetRegistrationForTests(): void {
-  registered = false;
-}
-
-/**
- * Group discovered generator models by provider, attaching the provider's
- * baseUrl so the adapter knows where to POST.
- */
-export function groupModelsByProvider(
-  models: ImageGenModel[],
-  baseUrlFor: (provider: string) => string | undefined,
-): Map<string, { baseUrl: string; models: ImageGenModel[] }> {
-  const grouped = new Map<string, { baseUrl: string; models: ImageGenModel[] }>();
-
-  for (const model of models) {
-    const baseUrl = model.baseUrl ?? baseUrlFor(model.provider);
-    // Without an endpoint the adapter cannot issue a request; skip rather than
-    // register a provider that is guaranteed to fail.
-    if (!baseUrl) continue;
-
-    let entry = grouped.get(model.provider);
-    if (!entry) {
-      entry = { baseUrl, models: [] };
-      grouped.set(model.provider, entry);
-    }
-    entry.models.push({ ...model, baseUrl });
-  }
-
-  return grouped;
-}
-
-/** Read a provider's baseUrl out of pi's registry. */
-function providerBaseUrlLookup(
-  registry: ChatModelRegistry,
-): (provider: string) => string | undefined {
-  const cache = new Map<string, string | undefined>();
-
-  return (provider: string) => {
-    if (cache.has(provider)) return cache.get(provider);
-
-    let baseUrl: string | undefined;
-    try {
-      const models = (registry.getAvailable?.() ?? registry.getAll()) as Array<{
-        provider?: string;
-        baseUrl?: string;
-      }>;
-      baseUrl = models.find((m) => m?.provider === provider && m.baseUrl)?.baseUrl;
-    } catch {
-      baseUrl = undefined;
-    }
-
-    cache.set(provider, baseUrl);
-    return baseUrl;
-  };
-}
-
-/**
- * Register every pi provider that looks capable of image generation into
- * pi-ai's images collection.
- *
- * Idempotent and best-effort: a failure here must never break the extension,
- * since generation still works for pi-ai's built-in providers.
- *
- * @returns the provider ids registered.
- */
-export async function registerRegistryImageProviders(
-  registry: ChatModelRegistry | undefined,
-  options?: { force?: boolean },
-): Promise<string[]> {
-  if (!registry) return [];
-  if (registered && !options?.force) return [];
-
-  const images = await getImagesModels();
-  if (!images) return [];
-
-  // `setProvider` is on MutableImagesModels; the built-in collection provides
-  // it, but guard in case a future pi-ai hands back an immutable one.
-  const mutable = images as unknown as {
-    setProvider?: (provider: unknown) => void;
-    getProvider?: (id: string) => unknown;
-  };
-  if (typeof mutable.setProvider !== "function") return [];
-
-  let createImagesProvider: CreateImagesProviderFn;
-  try {
-    const mod = (await import("@earendil-works/pi-ai")) as unknown as {
-      createImagesProvider?: CreateImagesProviderFn;
-    };
-    if (typeof mod.createImagesProvider !== "function") return [];
-    createImagesProvider = mod.createImagesProvider;
-  } catch {
-    return [];
-  }
-
-  const discovered = listRegistryImageGenModels(registry);
-  if (discovered.length === 0) {
-    registered = true;
-    return [];
-  }
-
-  const grouped = groupModelsByProvider(discovered, providerBaseUrlLookup(registry));
-  const added: string[] = [];
-
-  for (const [providerId, { models }] of grouped) {
-    // Never shadow a provider pi-ai serves natively — its own implementation
-    // is better informed than our generic adapter.
-    try {
-      if (mutable.getProvider?.(providerId)) continue;
-    } catch {
-      // Treat a lookup failure as "not present" and attempt registration.
-    }
-
-    try {
-      const provider = createImagesProvider({
-        id: providerId,
-        name: providerId,
-        models,
-        api: imagesApi,
-        auth: {
-          apiKey: {
-            name: `${providerId} API key`,
-            // Resolve through pi's own auth storage so the user never logs in
-            // twice. `resolve` MUST return an AuthResult (`{ auth: {...} }`);
-            // returning a bare key fails silently at request time.
-            resolve: async () => {
-              const key = await resolveProviderKey(registry, providerId);
-              return key ? { auth: { apiKey: key }, source: `pi:${providerId}` } : undefined;
-            },
-          },
-        },
-      });
-
-      mutable.setProvider(provider);
-      added.push(providerId);
-    } catch {
-      // One bad provider must not stop the rest.
-    }
-  }
-
-  registered = true;
-  return added;
-}
-
-/** Resolve a provider key from pi's auth storage, falling back to the env. */
-async function resolveProviderKey(
-  registry: ChatModelRegistry,
-  provider: string,
-): Promise<string | undefined> {
-  try {
-    const key = await registry.getApiKeyForProvider?.(provider);
-    if (key) return key;
-  } catch {
-    // Fall through to the environment.
-  }
-  const envName = `${provider.toUpperCase().replace(/[^A-Z0-9]/g, "_")}_API_KEY`;
-  return process.env[envName] || undefined;
-}

package/packages/image/src/settings.ts

@@ -1,173 +0,0 @@
-/**
- * @pi-unipi/image — Settings
- *
- * Config lives at `~/.unipi/config/image/config.json`. Every read is
- * try/catch-to-defaults so a corrupt file can never break the tools.
- *
- * The config directory is overridable via `UNIPI_IMAGE_CONFIG_DIR` so tests
- * do not have to reach into the real home directory.
- */
-
-import * as fs from "node:fs";
-import * as os from "node:os";
-import * as path from "node:path";
-
-/** Default system prompt for image recognition. */
-export const DEFAULT_RECOGNIZE_SYSTEM_PROMPT =
-  "You are a precise image analyst assisting a software engineer. " +
-  "Describe what is actually visible — never speculate about what is not shown. " +
-  "For screenshots, transcribe visible text, UI structure, and any errors verbatim. " +
-  "For diagrams, describe the components and their relationships. " +
-  "For photographs, describe the subject, setting, and notable detail. " +
-  "Be specific and concise; lead with the single most important observation.";
-
-/** Default image-generation model. */
-export const DEFAULT_GENERATE_MODEL = "openrouter/google/gemini-3-pro-image";
-
-export interface GenerateSettings {
-  /** Whether the image_generate tool is registered. */
-  enabled: boolean;
-  /** Model as "provider/model-id". */
-  model: string;
-  /** Directory for saved images. `~` is expanded. */
-  outputDir: string;
-  /** Whether to also write generated images to disk. */
-  saveToDisk: boolean;
-}
-
-export interface RecognizeSettings {
-  /** Whether the image_recognize tool is registered. */
-  enabled: boolean;
-  /** Model as "provider/model-id". Empty = use the session's current model. */
-  model: string;
-  /** System prompt sent with every recognition request. */
-  systemPrompt: string;
-}
-
-export interface ImageConfig {
-  generate: GenerateSettings;
-  recognize: RecognizeSettings;
-}
-
-export const DEFAULT_CONFIG: ImageConfig = {
-  generate: {
-    enabled: true,
-    model: DEFAULT_GENERATE_MODEL,
-    outputDir: "~/.unipi/images",
-    saveToDisk: true,
-  },
-  recognize: {
-    enabled: true,
-    model: "",
-    systemPrompt: DEFAULT_RECOGNIZE_SYSTEM_PROMPT,
-  },
-};
-
-/** Resolve the config directory, honouring the test override. */
-import { getSettings, registerSettings, setSettings } from "@pi-unipi/core";
-
-// Registered with the unified settings hub — canonical path already matched.
-registerSettings({
-  namespace: "image",
-  label: "Image",
-  defaults: DEFAULT_CONFIG as unknown as Record<string, unknown>,
-  schema: [
-    {
-      title: "Generate",
-      fields: [
-        { key: "generate.enabled", type: "boolean", label: "Generate enabled", description: "Register the image_generate tool" },
-        { key: "generate.model", type: "model", label: "Generate model", description: "Image generation model", presets: [DEFAULT_GENERATE_MODEL, "openrouter/openai/gpt-5-image", "openrouter/black-forest-labs/flux.2-pro"] },
-        { key: "generate.outputDir", type: "string", label: "Output dir", description: "~ expanded" },
-        { key: "generate.saveToDisk", type: "boolean", label: "Save to disk" },
-      ],
-    },
-    {
-      title: "Recognize",
-      fields: [
-        { key: "recognize.enabled", type: "boolean", label: "Recognize enabled", description: "Register the image_recognize tool" },
-        { key: "recognize.model", type: "model", label: "Recognize model", emptyLabel: "inherit (session model)", capability: "image-input", emptyOption: "inherit (session model)" },
-        { key: "recognize.systemPrompt", type: "string", label: "System prompt", description: "Sent with every recognition request", emptyLabel: "built-in analyst prompt" },
-      ],
-    },
-  ],
-});
-
-export function getConfigDir(): string {
-  const override = process.env.UNIPI_IMAGE_CONFIG_DIR;
-  if (override && override.trim().length > 0) return override;
-  return path.join(os.homedir(), ".unipi", "config", "image");
-}
-
-function getConfigPath(): string {
-  return path.join(getConfigDir(), "config.json");
-}
-
-/** Expand a leading `~` to the user's home directory. */
-export function expandHome(target: string): string {
-  if (target === "~") return os.homedir();
-  if (target.startsWith("~/") || target.startsWith("~\\")) {
-    return path.join(os.homedir(), target.slice(2));
-  }
-  return target;
-}
-
-function isRecord(value: unknown): value is Record<string, unknown> {
-  return value !== null && typeof value === "object" && !Array.isArray(value);
-}
-
-/** Merge a loaded section over its defaults, ignoring wrong-typed fields. */
-function mergeSection<T extends object>(defaults: T, loaded: unknown): T {
-  if (!isRecord(loaded)) return { ...defaults };
-
-  const merged: T = { ...defaults };
-  for (const key of Object.keys(defaults) as Array<keyof T & string>) {
-    const value = loaded[key];
-    if (value === undefined || value === null) continue;
-    // Only accept a value whose type matches the default's.
-    if (typeof value === typeof defaults[key]) {
-      merged[key] = value as T[keyof T & string];
-    }
-  }
-  return merged;
-}
-
-/** Load config (engine-layered), falling back to defaults on any problem. */
-export function loadConfig(): ImageConfig {
-  try {
-    const parsed: unknown = getSettings("image", process.cwd());
-    if (!isRecord(parsed)) return structuredClone(DEFAULT_CONFIG);
-    return {
-      generate: mergeSection(DEFAULT_CONFIG.generate, parsed.generate),
-      recognize: mergeSection(DEFAULT_CONFIG.recognize, parsed.recognize),
-    };
-  } catch {
-    return structuredClone(DEFAULT_CONFIG);
-  }
-}
-
-/** Persist config. Returns false instead of throwing when the write fails. */
-export function saveConfig(config: ImageConfig): boolean {
-  try {
-    setSettings("image", config as unknown as Record<string, unknown>, "global", process.cwd());
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-/** Apply a partial update, merging one level deep. */
-export function updateConfig(partial: Partial<ImageConfig>): ImageConfig {
-  const current = loadConfig();
-  const next: ImageConfig = {
-    generate: { ...current.generate, ...partial.generate },
-    recognize: { ...current.recognize, ...partial.recognize },
-  };
-  saveConfig(next);
-  return next;
-}
-
-/** Resolved absolute output directory for generated images. */
-export function getOutputDir(config: ImageConfig = loadConfig()): string {
-  const dir = config.generate.outputDir?.trim();
-  return expandHome(dir && dir.length > 0 ? dir : DEFAULT_CONFIG.generate.outputDir);
-}