@oh-my-pi/pi-ai 18.2.7 → 18.2.9

package/src/images/openai-hosted.ts

@@ -0,0 +1,185 @@
+import type { Api, Model } from "@oh-my-pi/pi-catalog/types";
+import {
+	applyCodexResidencyHeader,
+	CODEX_BASE_URL,
+	getCodexAccountId,
+	OPENAI_HEADER_VALUES,
+	OPENAI_HEADERS,
+	URL_PATHS,
+} from "@oh-my-pi/pi-catalog/wire/codex";
+import { parseImageMetadata, readSseJson, USER_AGENT } from "@oh-my-pi/pi-utils";
+import { withAuth } from "../auth-retry";
+import * as AIError from "../error";
+import { errorMessage, ImageApiError, modelHeaders, resolveOpenAIImageSize, toDataUrl, usageFromWire } from "./shared";
+import type { GeneratedImage, ImageGenerationOptions, ImageGenerationRequest, ImageGenerationResult } from "./types";
+
+const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
+const IMAGE_SYSTEM_INSTRUCTION =
+	"You are an AI image generator. Generate images based on user descriptions. Focus on creating high-quality, visually appealing images that match the user's request.";
+
+interface HostedOutput {
+	type: "image_generation_call" | "message";
+	result?: string;
+	content?: Array<{ type?: string; text?: string; refusal?: string }>;
+}
+
+interface HostedResponse {
+	output?: HostedOutput[];
+	usage?: { input_tokens?: number; output_tokens?: number };
+	error?: { message?: string };
+}
+
+interface HostedEvent {
+	type?: string;
+	item?: HostedOutput;
+	response?: HostedResponse;
+	error?: { message?: string };
+	message?: string;
+}
+
+function responsesUrl(model: Model<Api>): string {
+	const fallback =
+		model.api === "openai-codex-responses" || model.provider === "openai-codex"
+			? CODEX_BASE_URL
+			: DEFAULT_OPENAI_BASE_URL;
+	const baseUrl = (model.baseUrl || fallback).replace(/\/+$/, "");
+	if (model.api !== "openai-codex-responses" && model.provider !== "openai-codex") {
+		return `${baseUrl}/responses`;
+	}
+	const baseWithSlash = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`;
+	return new URL(URL_PATHS.RESPONSES.slice(1), baseWithSlash)
+		.toString()
+		.replace(URL_PATHS.RESPONSES, URL_PATHS.CODEX_RESPONSES);
+}
+
+async function headers(
+	carrier: Model<Api>,
+	key: string,
+	sessionId: string | undefined,
+	signal?: AbortSignal,
+): Promise<Headers> {
+	const result = new Headers(await modelHeaders(carrier, signal));
+	result.set("Content-Type", "application/json");
+	result.set("Authorization", `Bearer ${key}`);
+	if (carrier.api === "openai-codex-responses" || carrier.provider === "openai-codex") {
+		const accountId = getCodexAccountId(key);
+		result.delete("x-api-key");
+		if (accountId) result.set(OPENAI_HEADERS.ACCOUNT_ID, accountId);
+		applyCodexResidencyHeader(result, key);
+		result.set(OPENAI_HEADERS.BETA, OPENAI_HEADER_VALUES.BETA_RESPONSES);
+		result.set(OPENAI_HEADERS.ORIGINATOR, OPENAI_HEADER_VALUES.ORIGINATOR_CODEX);
+		result.set("User-Agent", USER_AGENT);
+		if (sessionId) {
+			result.set(OPENAI_HEADERS.CONVERSATION_ID, sessionId);
+			result.set(OPENAI_HEADERS.SESSION_ID, sessionId);
+		}
+	}
+	return result;
+}
+
+function collectResponse(response: HostedResponse): ImageGenerationResult {
+	const images: GeneratedImage[] = [];
+	const texts: string[] = [];
+	for (const output of response.output ?? []) {
+		if (output.type === "image_generation_call" && output.result) {
+			const bytes = Buffer.from(output.result, "base64");
+			images.push({ data: output.result, mimeType: parseImageMetadata(bytes)?.mimeType ?? "image/webp" });
+		}
+		if (output.type === "message") {
+			for (const part of output.content ?? []) {
+				if (part.type === "output_text" && part.text) texts.push(part.text);
+				if (part.type === "refusal" && part.refusal) texts.push(part.refusal);
+			}
+		}
+	}
+	const text = texts.join("\n").trim();
+	return { images, ...(text ? { text } : {}), usage: usageFromWire(response.usage) };
+}
+
+async function parseSse(response: Response, signal?: AbortSignal): Promise<ImageGenerationResult> {
+	if (!response.body) {
+		throw new AIError.ProviderResponseError("OpenAI hosted image response has no body", { kind: "empty-body" });
+	}
+	const fallbackOutput: HostedOutput[] = [];
+	let completed: HostedResponse | undefined;
+	for await (const event of readSseJson<HostedEvent>(response.body, signal)) {
+		if (event.type === "error") {
+			throw new AIError.ProviderResponseError(
+				event.error?.message ?? event.message ?? "OpenAI image request failed",
+			);
+		}
+		if (event.type === "response.failed") {
+			throw new AIError.ProviderResponseError(event.response?.error?.message ?? "OpenAI image request failed");
+		}
+		if (event.type === "response.output_item.done" && event.item) fallbackOutput.push(event.item);
+		if ((event.type === "response.completed" || event.type === "response.done") && event.response) {
+			completed = event.response;
+		}
+	}
+	return collectResponse(completed?.output?.length ? completed : { output: fallbackOutput, usage: completed?.usage });
+}
+
+export async function generateHostedImage(
+	model: Model,
+	request: ImageGenerationRequest,
+	options: ImageGenerationOptions,
+): Promise<ImageGenerationResult> {
+	const carrier = options.carrier;
+	if (!carrier) throw new AIError.ValidationError("Hosted image generation requires an explicit carrier model");
+	if (carrier.api !== "openai-responses" && carrier.api !== "openai-codex-responses") {
+		throw new AIError.ValidationError(`Hosted image carrier API ${carrier.api} is unsupported`);
+	}
+	const stream = carrier.api === "openai-codex-responses";
+	const content: Array<Record<string, unknown>> = [{ type: "input_text", text: request.prompt }];
+	for (const image of request.inputImages ?? []) {
+		content.push({ type: "input_image", detail: "auto", image_url: toDataUrl(image) });
+	}
+	const size = resolveOpenAIImageSize(request.aspectRatio, request.imageSize);
+	const tool = {
+		type: "image_generation",
+		action: content.length > 1 ? "edit" : "generate",
+		output_format: "webp",
+		...(size ? { size } : {}),
+		...(model.api === "openai-responses" ? { model: model.requestModelId ?? model.id } : {}),
+	};
+	const body = {
+		model: carrier.requestModelId ?? carrier.id,
+		input: [{ role: "user", content }],
+		tools: [tool],
+		tool_choice: { type: "image_generation" },
+		store: false,
+		...(stream ? { instructions: IMAGE_SYSTEM_INSTRUCTION, stream: true } : {}),
+	};
+	const fetchImpl = options.fetch ?? fetch;
+	return withAuth(
+		options.apiKey,
+		async key => {
+			const response = await fetchImpl(responsesUrl(carrier), {
+				method: "POST",
+				headers: await headers(carrier, key, options.sessionId, options.signal),
+				body: JSON.stringify(body),
+				signal: options.signal,
+			});
+			if (!response.ok) {
+				const text = await response.text();
+				throw new ImageApiError(
+					`OpenAI image request failed (${response.status}): ${errorMessage(text)}`,
+					response.status,
+					{ headers: response.headers },
+				);
+			}
+			if (stream || response.headers.get("content-type")?.includes("text/event-stream")) {
+				return parseSse(response, options.signal);
+			}
+			const value = (await response.json()) as HostedResponse;
+			if (value.error) throw new AIError.ProviderResponseError(value.error.message ?? "OpenAI image request failed");
+			if (!Array.isArray(value.output)) {
+				throw new AIError.ProviderResponseError("OpenAI image request returned a malformed response", {
+					kind: "envelope",
+				});
+			}
+			return collectResponse(value);
+		},
+		{ signal: options.signal },
+	);
+}

package/src/images/openai-images.ts

@@ -0,0 +1,110 @@
+import type { Model } from "@oh-my-pi/pi-catalog/types";
+import * as AIError from "../error";
+import {
+	decodeImageResponse,
+	imageBaseUrl,
+	postJson,
+	postMultipart,
+	resolveOpenAIImageSize,
+	toDataUrl,
+} from "./shared";
+import type { ImageGenerationOptions, ImageGenerationRequest, ImageGenerationResult } from "./types";
+
+export const XAI_MAX_EDIT_IMAGES = 3;
+
+export function resolveXAIResolution(imageSize?: string): "1k" | "2k" {
+	return !imageSize || imageSize === "1024x1024" ? "1k" : "2k";
+}
+
+export async function generateOpenAIImage(
+	model: Model,
+	request: ImageGenerationRequest,
+	options: ImageGenerationOptions,
+): Promise<ImageGenerationResult> {
+	const fetchImpl = options.fetch ?? fetch;
+	const size = resolveOpenAIImageSize(request.aspectRatio, request.imageSize);
+	const count = request.count ?? 1;
+	const isXAI = model.provider === "xai" || model.provider === "xai-oauth";
+	const generationBody = isXAI
+		? {
+				model: model.requestModelId ?? model.id,
+				prompt: request.prompt,
+				aspect_ratio: request.aspectRatio ?? "1:1",
+				resolution: resolveXAIResolution(request.imageSize),
+				n: count,
+				response_format: "b64_json",
+			}
+		: {
+				model: model.requestModelId ?? model.id,
+				prompt: request.prompt,
+				n: count,
+				response_format: "b64_json",
+				...(size ? { size } : {}),
+			};
+	const references = (request.inputImages ?? []).map(image => ({ type: "image_url", url: toDataUrl(image) }));
+	if (isXAI && references.length > XAI_MAX_EDIT_IMAGES) {
+		throw new AIError.ValidationError(
+			`${model.provider} image edits accept up to ${XAI_MAX_EDIT_IMAGES} reference images; got ${references.length}`,
+		);
+	}
+	const [firstReference, ...remainingReferences] = references;
+	const body = isXAI
+		? remainingReferences.length === 0
+			? { ...generationBody, image: firstReference }
+			: { ...generationBody, images: references }
+		: { ...generationBody, input_references: references };
+	const baseUrl = imageBaseUrl(model);
+	let response: unknown;
+	if (references.length === 0) {
+		response = await postJson({
+			model,
+			url: `${baseUrl}/images/generations`,
+			body: generationBody,
+			apiKey: options.apiKey,
+			fetch: fetchImpl,
+			signal: options.signal,
+		});
+	} else {
+		try {
+			if (model.provider === "openai") {
+				const form = new FormData();
+				form.set("model", model.requestModelId ?? model.id);
+				form.set("prompt", request.prompt);
+				form.set("n", String(count));
+				form.set("response_format", "b64_json");
+				if (size) form.set("size", size);
+				for (const image of request.inputImages ?? []) {
+					form.append("image", new File([Buffer.from(image.data, "base64")], "image", { type: image.mimeType }));
+				}
+				response = await postMultipart({
+					model,
+					url: `${baseUrl}/images/edits`,
+					body: form,
+					apiKey: options.apiKey,
+					fetch: fetchImpl,
+					signal: options.signal,
+				});
+			} else {
+				response = await postJson({
+					model,
+					url: `${baseUrl}/images/edits`,
+					body,
+					apiKey: options.apiKey,
+					fetch: fetchImpl,
+					signal: options.signal,
+				});
+			}
+		} catch (error) {
+			if (!(error instanceof AIError.ProviderHttpError) || error.status !== 404) throw error;
+			response = await postJson({
+				model,
+				url: `${baseUrl}/images/generations`,
+				body,
+				apiKey: options.apiKey,
+				fetch: fetchImpl,
+				signal: options.signal,
+			});
+		}
+	}
+	return decodeImageResponse(response, fetchImpl, options.signal);
+}

package/src/images/openrouter-images.ts

@@ -0,0 +1,33 @@
+import type { Model } from "@oh-my-pi/pi-catalog/types";
+import { decodeImageResponse, imageBaseUrl, postJson, toDataUrl } from "./shared";
+import type { ImageGenerationOptions, ImageGenerationRequest, ImageGenerationResult } from "./types";
+
+export async function generateOpenRouterImage(
+	model: Model,
+	request: ImageGenerationRequest,
+	options: ImageGenerationOptions,
+): Promise<ImageGenerationResult> {
+	const fetchImpl = options.fetch ?? fetch;
+	const inputReferences = (request.inputImages ?? []).map(image => ({
+		type: "image_url",
+		image_url: { url: toDataUrl(image) },
+	}));
+	const body = {
+		model: model.requestModelId ?? model.id,
+		prompt: request.prompt,
+		n: request.count ?? 1,
+		response_format: "b64_json",
+		...(request.aspectRatio ? { aspect_ratio: request.aspectRatio } : {}),
+		...(request.imageSize ? { image_size: request.imageSize } : {}),
+		...(inputReferences.length > 0 ? { input_references: inputReferences } : {}),
+	};
+	const response = await postJson({
+		model,
+		url: `${imageBaseUrl(model)}/images`,
+		body,
+		apiKey: options.apiKey,
+		fetch: fetchImpl,
+		signal: options.signal,
+	});
+	return decodeImageResponse(response, fetchImpl, options.signal);
+}

package/src/images/shared.ts

@@ -0,0 +1,193 @@
+import type { FetchImpl, Model, Usage } from "@oh-my-pi/pi-catalog/types";
+import { parseImageMetadata, USER_AGENT } from "@oh-my-pi/pi-utils";
+import type { ApiKey } from "../auth-retry";
+import { withAuth } from "../auth-retry";
+import * as AIError from "../error";
+import type { GeneratedImage } from "./types";
+
+export class ImageApiError extends AIError.ProviderHttpError {
+	override readonly name = "ImageApiError";
+}
+
+export function emptyUsage(input = 0, output = 0, cost = 0): Usage {
+	return {
+		input,
+		output,
+		cacheRead: 0,
+		cacheWrite: 0,
+		totalTokens: input + output,
+		cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: cost },
+	};
+}
+
+export function usageFromWire(value: unknown): Usage {
+	if (value === null || typeof value !== "object") return emptyUsage();
+	const usage = value as Record<string, unknown>;
+	const number = (key: string): number => (typeof usage[key] === "number" ? usage[key] : 0);
+	return emptyUsage(
+		number("input_tokens") || number("prompt_tokens"),
+		number("output_tokens") || number("completion_tokens"),
+		number("cost"),
+	);
+}
+
+export function imageBaseUrl(model: Model): string {
+	if (!model.baseUrl) throw new AIError.ValidationError(`Image model ${model.provider}/${model.id} has no base URL`);
+	return model.baseUrl.replace(/\/+$/, "");
+}
+
+export async function modelHeaders(model: Model, signal?: AbortSignal): Promise<Record<string, string>> {
+	return { ...model.headers, ...(await model.resolveHeaders?.(signal)) };
+}
+
+export function errorMessage(rawText: string): string {
+	try {
+		const parsed = JSON.parse(rawText) as { detail?: string; error?: { message?: string } };
+		return parsed.detail ?? parsed.error?.message ?? rawText;
+	} catch {
+		return rawText;
+	}
+}
+
+async function parseImageApiResponse(model: Model, response: Response): Promise<unknown> {
+	const text = await response.text();
+	if (!response.ok) {
+		throw new ImageApiError(
+			`${model.provider}/${model.id} image request failed (${response.status}): ${errorMessage(text)}`,
+			response.status,
+			{ headers: response.headers },
+		);
+	}
+	try {
+		return JSON.parse(text) as unknown;
+	} catch (cause) {
+		throw new AIError.ProviderResponseError("Image API returned malformed JSON", {
+			provider: model.provider,
+			kind: "envelope",
+			cause,
+		});
+	}
+}
+
+export async function postJson(options: {
+	model: Model;
+	url: string;
+	body: unknown;
+	apiKey: ApiKey;
+	fetch: FetchImpl;
+	signal?: AbortSignal;
+}): Promise<unknown> {
+	return withAuth(
+		options.apiKey,
+		async key => {
+			const response = await options.fetch(options.url, {
+				method: "POST",
+				headers: {
+					...(await modelHeaders(options.model, options.signal)),
+					Authorization: `Bearer ${key}`,
+					"Content-Type": "application/json",
+					"User-Agent": USER_AGENT,
+				},
+				body: JSON.stringify(options.body),
+				signal: options.signal,
+			});
+			return parseImageApiResponse(options.model, response);
+		},
+		{ signal: options.signal },
+	);
+}
+
+export async function postMultipart(options: {
+	model: Model;
+	url: string;
+	body: FormData;
+	apiKey: ApiKey;
+	fetch: FetchImpl;
+	signal?: AbortSignal;
+}): Promise<unknown> {
+	return withAuth(
+		options.apiKey,
+		async key => {
+			const response = await options.fetch(options.url, {
+				method: "POST",
+				headers: {
+					...(await modelHeaders(options.model, options.signal)),
+					Authorization: `Bearer ${key}`,
+					"User-Agent": USER_AGENT,
+				},
+				body: options.body,
+				signal: options.signal,
+			});
+			return parseImageApiResponse(options.model, response);
+		},
+		{ signal: options.signal },
+	);
+}
+
+async function imageFromUrl(url: string, fetch: FetchImpl, signal?: AbortSignal): Promise<GeneratedImage> {
+	const response = await fetch(url, { signal });
+	if (!response.ok) {
+		const text = await response.text();
+		throw new ImageApiError(`Image download failed (${response.status}): ${text}`, response.status, {
+			headers: response.headers,
+		});
+	}
+	const mimeType = response.headers.get("content-type")?.split(";")[0];
+	if (!mimeType?.startsWith("image/")) {
+		throw new AIError.ProviderResponseError(`Image URL returned unsupported content type: ${mimeType ?? "missing"}`, {
+			kind: "envelope",
+		});
+	}
+	const bytes = new Uint8Array(await response.arrayBuffer());
+	return { data: bytes.toBase64(), mimeType };
+}
+
+export async function decodeImageResponse(
+	value: unknown,
+	fetch: FetchImpl,
+	signal?: AbortSignal,
+): Promise<{ images: GeneratedImage[]; usage: Usage }> {
+	if (value === null || typeof value !== "object") {
+		throw new AIError.ProviderResponseError("Image API returned a malformed response", { kind: "envelope" });
+	}
+	const root = value as { data?: unknown; usage?: unknown };
+	if (!Array.isArray(root.data)) {
+		throw new AIError.ProviderResponseError("Image API response is missing data", { kind: "envelope" });
+	}
+	const images: GeneratedImage[] = [];
+	for (const item of root.data) {
+		if (item === null || typeof item !== "object") continue;
+		const image = item as { b64_json?: unknown; url?: unknown; media_type?: unknown };
+		if (typeof image.b64_json === "string" && image.b64_json.length > 0) {
+			const bytes = Buffer.from(image.b64_json, "base64");
+			const mimeType =
+				typeof image.media_type === "string"
+					? image.media_type
+					: (parseImageMetadata(bytes)?.mimeType ?? "image/png");
+			images.push({ data: image.b64_json, mimeType });
+		} else if (typeof image.url === "string" && image.url.length > 0) {
+			images.push(await imageFromUrl(image.url, fetch, signal));
+		}
+	}
+	return { images, usage: usageFromWire(root.usage) };
+}
+
+export function toDataUrl(image: GeneratedImage): string {
+	return `data:${image.mimeType};base64,${image.data}`;
+}
+
+export function resolveOpenAIImageSize(aspectRatio?: string, imageSize?: string): string | undefined {
+	if (imageSize) return imageSize;
+	switch (aspectRatio) {
+		case "1:1":
+			return "1024x1024";
+		case "3:4":
+		case "9:16":
+			return "1024x1536";
+		case "4:3":
+		case "16:9":
+			return "1536x1024";
+		default:
+			return undefined;
+	}
+}

package/src/images/types.ts

@@ -0,0 +1,36 @@
+import type { Api, FetchImpl, Model, Usage } from "@oh-my-pi/pi-catalog/types";
+import type { ApiKey } from "../auth-retry";
+
+export interface ImageInput {
+	data: string;
+	mimeType: string;
+}
+
+export interface ImageGenerationRequest {
+	prompt: string;
+	inputImages?: ImageInput[];
+	aspectRatio?: string;
+	imageSize?: string;
+	count?: number;
+}
+
+export interface GeneratedImage {
+	data: string;
+	mimeType: string;
+}
+
+export interface ImageGenerationResult {
+	images: GeneratedImage[];
+	text?: string;
+	usage: Usage;
+}
+
+export interface ImageGenerationOptions {
+	apiKey: ApiKey;
+	fetch?: FetchImpl;
+	signal?: AbortSignal;
+	/** Chat model that carries a Responses `image_generation` tool call. */
+	carrier?: Model<Api>;
+	/** Stable provider session id, used by the Codex Responses carrier. */
+	sessionId?: string;
+}

package/src/index.ts

@@ -1,12 +1,18 @@
 export { type Type, type } from "@oh-my-pi/omptype";
 export * from "./api-registry";
 export type * from "./auth-broker";
-export type { AuthGatewayBootOptions, ModelResolver } from "./auth-gateway/server";
+export type { AuthGatewayBootOptions, ModelResolver } from "./auth-gateway/dispatch";
 export * from "./auth-gateway/types";
 export * from "./auth-retry";
 export * from "./auth-storage";
 export * from "./error/rate-limit";
+export * from "./embeddings";
+export * from "./images";
 export * from "./judgment";
+export * from "./rerank";
+export * from "./speech";
+export * from "./transcription";
+export * from "./video";
 export * from "./oneshot-retry";
 export * from "./provider-details";
 export * from "./provider-session-state";
@@ -34,6 +40,7 @@ export * from "./stream";
 export * from "./types";
 export * from "./usage";
 export * from "./usage/claude";
+export * from "./usage/claude-reset";
 export * from "./usage/cursor";
 export * from "./usage/gemini";
 export * from "./usage/github-copilot";

package/src/judgment/typesafe.ts

@@ -66,6 +66,8 @@ export interface TypeSafeJudgeOptions {
 	baseUrl?: string;
 	/** Defaults to {@link typesafeModel}. */
 	model?: string;
+	/** Static headers attached to judgment requests (e.g. proxy routing, gateway auth). */
+	headers?: Record<string, string>;
 	fetch?: FetchImpl;
 	/** Per-attempt timeout; defaults to {@link DEFAULT_TIMEOUT_MS}. */
 	timeoutMs?: number;
@@ -102,6 +104,7 @@ export class TypeSafeJudge implements Judge {
 	readonly model: string;
 	readonly baseUrl: string;
 	readonly #apiKey: ApiKey;
+	readonly #headers: Record<string, string> | undefined;
 	readonly #fetch: FetchImpl;
 	readonly #timeoutMs: number;
 
@@ -111,6 +114,7 @@ export class TypeSafeJudge implements Judge {
 		this.provider = options.provider ?? TYPESAFE_PROVIDER;
 		this.baseUrl = (options.baseUrl ?? typesafeBaseUrl()).replace(/\/+$/, "");
 		this.model = options.model ?? typesafeModel();
+		this.#headers = options.headers;
 		this.#fetch = options.fetch ?? fetch;
 		this.#timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
 		this.label = `${this.provider}/${this.model}`;
@@ -145,6 +149,7 @@ export class TypeSafeJudge implements Judge {
 	async #attempt<T>(path: string, body: string, key: string, signal: AbortSignal | undefined): Promise<T> {
 		const url = `${this.baseUrl}${path}`;
 		const headers: Record<string, string> = {
+			...this.#headers,
 			Authorization: `Bearer ${key}`,
 			Accept: "application/json",
 			"Content-Type": "application/json",