@oh-my-pi/pi-ai 18.2.7 → 18.2.8

package/src/images/openai-images.ts

@@ -0,0 +1,110 @@
+import type { Model } from "@oh-my-pi/pi-catalog/types";
+import * as AIError from "../error";
+import {
+	decodeImageResponse,
+	imageBaseUrl,
+	postJson,
+	postMultipart,
+	resolveOpenAIImageSize,
+	toDataUrl,
+} from "./shared";
+import type { ImageGenerationOptions, ImageGenerationRequest, ImageGenerationResult } from "./types";
+
+export const XAI_MAX_EDIT_IMAGES = 3;
+
+export function resolveXAIResolution(imageSize?: string): "1k" | "2k" {
+	return !imageSize || imageSize === "1024x1024" ? "1k" : "2k";
+}
+
+export async function generateOpenAIImage(
+	model: Model,
+	request: ImageGenerationRequest,
+	options: ImageGenerationOptions,
+): Promise<ImageGenerationResult> {
+	const fetchImpl = options.fetch ?? fetch;
+	const size = resolveOpenAIImageSize(request.aspectRatio, request.imageSize);
+	const count = request.count ?? 1;
+	const isXAI = model.provider === "xai" || model.provider === "xai-oauth";
+	const generationBody = isXAI
+		? {
+				model: model.requestModelId ?? model.id,
+				prompt: request.prompt,
+				aspect_ratio: request.aspectRatio ?? "1:1",
+				resolution: resolveXAIResolution(request.imageSize),
+				n: count,
+				response_format: "b64_json",
+			}
+		: {
+				model: model.requestModelId ?? model.id,
+				prompt: request.prompt,
+				n: count,
+				response_format: "b64_json",
+				...(size ? { size } : {}),
+			};
+	const references = (request.inputImages ?? []).map(image => ({ type: "image_url", url: toDataUrl(image) }));
+	if (isXAI && references.length > XAI_MAX_EDIT_IMAGES) {
+		throw new AIError.ValidationError(
+			`${model.provider} image edits accept up to ${XAI_MAX_EDIT_IMAGES} reference images; got ${references.length}`,
+		);
+	}
+	const [firstReference, ...remainingReferences] = references;
+	const body = isXAI
+		? remainingReferences.length === 0
+			? { ...generationBody, image: firstReference }
+			: { ...generationBody, images: references }
+		: { ...generationBody, input_references: references };
+	const baseUrl = imageBaseUrl(model);
+	let response: unknown;
+	if (references.length === 0) {
+		response = await postJson({
+			model,
+			url: `${baseUrl}/images/generations`,
+			body: generationBody,
+			apiKey: options.apiKey,
+			fetch: fetchImpl,
+			signal: options.signal,
+		});
+	} else {
+		try {
+			if (model.provider === "openai") {
+				const form = new FormData();
+				form.set("model", model.requestModelId ?? model.id);
+				form.set("prompt", request.prompt);
+				form.set("n", String(count));
+				form.set("response_format", "b64_json");
+				if (size) form.set("size", size);
+				for (const image of request.inputImages ?? []) {
+					form.append("image", new File([Buffer.from(image.data, "base64")], "image", { type: image.mimeType }));
+				}
+				response = await postMultipart({
+					model,
+					url: `${baseUrl}/images/edits`,
+					body: form,
+					apiKey: options.apiKey,
+					fetch: fetchImpl,
+					signal: options.signal,
+				});
+			} else {
+				response = await postJson({
+					model,
+					url: `${baseUrl}/images/edits`,
+					body,
+					apiKey: options.apiKey,
+					fetch: fetchImpl,
+					signal: options.signal,
+				});
+			}
+		} catch (error) {
+			if (!(error instanceof AIError.ProviderHttpError) || error.status !== 404) throw error;
+			response = await postJson({
+				model,
+				url: `${baseUrl}/images/generations`,
+				body,
+				apiKey: options.apiKey,
+				fetch: fetchImpl,
+				signal: options.signal,
+			});
+		}
+	}
+	return decodeImageResponse(response, fetchImpl, options.signal);
+}

package/src/images/openrouter-images.ts

@@ -0,0 +1,33 @@
+import type { Model } from "@oh-my-pi/pi-catalog/types";
+import { decodeImageResponse, imageBaseUrl, postJson, toDataUrl } from "./shared";
+import type { ImageGenerationOptions, ImageGenerationRequest, ImageGenerationResult } from "./types";
+
+export async function generateOpenRouterImage(
+	model: Model,
+	request: ImageGenerationRequest,
+	options: ImageGenerationOptions,
+): Promise<ImageGenerationResult> {
+	const fetchImpl = options.fetch ?? fetch;
+	const inputReferences = (request.inputImages ?? []).map(image => ({
+		type: "image_url",
+		image_url: { url: toDataUrl(image) },
+	}));
+	const body = {
+		model: model.requestModelId ?? model.id,
+		prompt: request.prompt,
+		n: request.count ?? 1,
+		response_format: "b64_json",
+		...(request.aspectRatio ? { aspect_ratio: request.aspectRatio } : {}),
+		...(request.imageSize ? { image_size: request.imageSize } : {}),
+		...(inputReferences.length > 0 ? { input_references: inputReferences } : {}),
+	};
+	const response = await postJson({
+		model,
+		url: `${imageBaseUrl(model)}/images`,
+		body,
+		apiKey: options.apiKey,
+		fetch: fetchImpl,
+		signal: options.signal,
+	});
+	return decodeImageResponse(response, fetchImpl, options.signal);
+}

package/src/images/shared.ts

@@ -0,0 +1,193 @@
+import type { FetchImpl, Model, Usage } from "@oh-my-pi/pi-catalog/types";
+import { parseImageMetadata, USER_AGENT } from "@oh-my-pi/pi-utils";
+import type { ApiKey } from "../auth-retry";
+import { withAuth } from "../auth-retry";
+import * as AIError from "../error";
+import type { GeneratedImage } from "./types";
+
+export class ImageApiError extends AIError.ProviderHttpError {
+	override readonly name = "ImageApiError";
+}
+
+export function emptyUsage(input = 0, output = 0, cost = 0): Usage {
+	return {
+		input,
+		output,
+		cacheRead: 0,
+		cacheWrite: 0,
+		totalTokens: input + output,
+		cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: cost },
+	};
+}
+
+export function usageFromWire(value: unknown): Usage {
+	if (value === null || typeof value !== "object") return emptyUsage();
+	const usage = value as Record<string, unknown>;
+	const number = (key: string): number => (typeof usage[key] === "number" ? usage[key] : 0);
+	return emptyUsage(
+		number("input_tokens") || number("prompt_tokens"),
+		number("output_tokens") || number("completion_tokens"),
+		number("cost"),
+	);
+}
+
+export function imageBaseUrl(model: Model): string {
+	if (!model.baseUrl) throw new AIError.ValidationError(`Image model ${model.provider}/${model.id} has no base URL`);
+	return model.baseUrl.replace(/\/+$/, "");
+}
+
+export async function modelHeaders(model: Model, signal?: AbortSignal): Promise<Record<string, string>> {
+	return { ...model.headers, ...(await model.resolveHeaders?.(signal)) };
+}
+
+export function errorMessage(rawText: string): string {
+	try {
+		const parsed = JSON.parse(rawText) as { detail?: string; error?: { message?: string } };
+		return parsed.detail ?? parsed.error?.message ?? rawText;
+	} catch {
+		return rawText;
+	}
+}
+
+async function parseImageApiResponse(model: Model, response: Response): Promise<unknown> {
+	const text = await response.text();
+	if (!response.ok) {
+		throw new ImageApiError(
+			`${model.provider}/${model.id} image request failed (${response.status}): ${errorMessage(text)}`,
+			response.status,
+			{ headers: response.headers },
+		);
+	}
+	try {
+		return JSON.parse(text) as unknown;
+	} catch (cause) {
+		throw new AIError.ProviderResponseError("Image API returned malformed JSON", {
+			provider: model.provider,
+			kind: "envelope",
+			cause,
+		});
+	}
+}
+
+export async function postJson(options: {
+	model: Model;
+	url: string;
+	body: unknown;
+	apiKey: ApiKey;
+	fetch: FetchImpl;
+	signal?: AbortSignal;
+}): Promise<unknown> {
+	return withAuth(
+		options.apiKey,
+		async key => {
+			const response = await options.fetch(options.url, {
+				method: "POST",
+				headers: {
+					...(await modelHeaders(options.model, options.signal)),
+					Authorization: `Bearer ${key}`,
+					"Content-Type": "application/json",
+					"User-Agent": USER_AGENT,
+				},
+				body: JSON.stringify(options.body),
+				signal: options.signal,
+			});
+			return parseImageApiResponse(options.model, response);
+		},
+		{ signal: options.signal },
+	);
+}
+
+export async function postMultipart(options: {
+	model: Model;
+	url: string;
+	body: FormData;
+	apiKey: ApiKey;
+	fetch: FetchImpl;
+	signal?: AbortSignal;
+}): Promise<unknown> {
+	return withAuth(
+		options.apiKey,
+		async key => {
+			const response = await options.fetch(options.url, {
+				method: "POST",
+				headers: {
+					...(await modelHeaders(options.model, options.signal)),
+					Authorization: `Bearer ${key}`,
+					"User-Agent": USER_AGENT,
+				},
+				body: options.body,
+				signal: options.signal,
+			});
+			return parseImageApiResponse(options.model, response);
+		},
+		{ signal: options.signal },
+	);
+}
+
+async function imageFromUrl(url: string, fetch: FetchImpl, signal?: AbortSignal): Promise<GeneratedImage> {
+	const response = await fetch(url, { signal });
+	if (!response.ok) {
+		const text = await response.text();
+		throw new ImageApiError(`Image download failed (${response.status}): ${text}`, response.status, {
+			headers: response.headers,
+		});
+	}
+	const mimeType = response.headers.get("content-type")?.split(";")[0];
+	if (!mimeType?.startsWith("image/")) {
+		throw new AIError.ProviderResponseError(`Image URL returned unsupported content type: ${mimeType ?? "missing"}`, {
+			kind: "envelope",
+		});
+	}
+	const bytes = new Uint8Array(await response.arrayBuffer());
+	return { data: bytes.toBase64(), mimeType };
+}
+
+export async function decodeImageResponse(
+	value: unknown,
+	fetch: FetchImpl,
+	signal?: AbortSignal,
+): Promise<{ images: GeneratedImage[]; usage: Usage }> {
+	if (value === null || typeof value !== "object") {
+		throw new AIError.ProviderResponseError("Image API returned a malformed response", { kind: "envelope" });
+	}
+	const root = value as { data?: unknown; usage?: unknown };
+	if (!Array.isArray(root.data)) {
+		throw new AIError.ProviderResponseError("Image API response is missing data", { kind: "envelope" });
+	}
+	const images: GeneratedImage[] = [];
+	for (const item of root.data) {
+		if (item === null || typeof item !== "object") continue;
+		const image = item as { b64_json?: unknown; url?: unknown; media_type?: unknown };
+		if (typeof image.b64_json === "string" && image.b64_json.length > 0) {
+			const bytes = Buffer.from(image.b64_json, "base64");
+			const mimeType =
+				typeof image.media_type === "string"
+					? image.media_type
+					: (parseImageMetadata(bytes)?.mimeType ?? "image/png");
+			images.push({ data: image.b64_json, mimeType });
+		} else if (typeof image.url === "string" && image.url.length > 0) {
+			images.push(await imageFromUrl(image.url, fetch, signal));
+		}
+	}
+	return { images, usage: usageFromWire(root.usage) };
+}
+
+export function toDataUrl(image: GeneratedImage): string {
+	return `data:${image.mimeType};base64,${image.data}`;
+}
+
+export function resolveOpenAIImageSize(aspectRatio?: string, imageSize?: string): string | undefined {
+	if (imageSize) return imageSize;
+	switch (aspectRatio) {
+		case "1:1":
+			return "1024x1024";
+		case "3:4":
+		case "9:16":
+			return "1024x1536";
+		case "4:3":
+		case "16:9":
+			return "1536x1024";
+		default:
+			return undefined;
+	}
+}

package/src/images/types.ts

@@ -0,0 +1,36 @@
+import type { Api, FetchImpl, Model, Usage } from "@oh-my-pi/pi-catalog/types";
+import type { ApiKey } from "../auth-retry";
+
+export interface ImageInput {
+	data: string;
+	mimeType: string;
+}
+
+export interface ImageGenerationRequest {
+	prompt: string;
+	inputImages?: ImageInput[];
+	aspectRatio?: string;
+	imageSize?: string;
+	count?: number;
+}
+
+export interface GeneratedImage {
+	data: string;
+	mimeType: string;
+}
+
+export interface ImageGenerationResult {
+	images: GeneratedImage[];
+	text?: string;
+	usage: Usage;
+}
+
+export interface ImageGenerationOptions {
+	apiKey: ApiKey;
+	fetch?: FetchImpl;
+	signal?: AbortSignal;
+	/** Chat model that carries a Responses `image_generation` tool call. */
+	carrier?: Model<Api>;
+	/** Stable provider session id, used by the Codex Responses carrier. */
+	sessionId?: string;
+}

package/src/index.ts

@@ -1,12 +1,18 @@
 export { type Type, type } from "@oh-my-pi/omptype";
 export * from "./api-registry";
 export type * from "./auth-broker";
-export type { AuthGatewayBootOptions, ModelResolver } from "./auth-gateway/server";
+export type { AuthGatewayBootOptions, ModelResolver } from "./auth-gateway/dispatch";
 export * from "./auth-gateway/types";
 export * from "./auth-retry";
 export * from "./auth-storage";
 export * from "./error/rate-limit";
+export * from "./embeddings";
+export * from "./images";
 export * from "./judgment";
+export * from "./rerank";
+export * from "./speech";
+export * from "./transcription";
+export * from "./video";
 export * from "./oneshot-retry";
 export * from "./provider-details";
 export * from "./provider-session-state";

package/src/judgment/typesafe.ts

@@ -66,6 +66,8 @@ export interface TypeSafeJudgeOptions {
 	baseUrl?: string;
 	/** Defaults to {@link typesafeModel}. */
 	model?: string;
+	/** Static headers attached to judgment requests (e.g. proxy routing, gateway auth). */
+	headers?: Record<string, string>;
 	fetch?: FetchImpl;
 	/** Per-attempt timeout; defaults to {@link DEFAULT_TIMEOUT_MS}. */
 	timeoutMs?: number;
@@ -102,6 +104,7 @@ export class TypeSafeJudge implements Judge {
 	readonly model: string;
 	readonly baseUrl: string;
 	readonly #apiKey: ApiKey;
+	readonly #headers: Record<string, string> | undefined;
 	readonly #fetch: FetchImpl;
 	readonly #timeoutMs: number;
 
@@ -111,6 +114,7 @@ export class TypeSafeJudge implements Judge {
 		this.provider = options.provider ?? TYPESAFE_PROVIDER;
 		this.baseUrl = (options.baseUrl ?? typesafeBaseUrl()).replace(/\/+$/, "");
 		this.model = options.model ?? typesafeModel();
+		this.#headers = options.headers;
 		this.#fetch = options.fetch ?? fetch;
 		this.#timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
 		this.label = `${this.provider}/${this.model}`;
@@ -145,6 +149,7 @@ export class TypeSafeJudge implements Judge {
 	async #attempt<T>(path: string, body: string, key: string, signal: AbortSignal | undefined): Promise<T> {
 		const url = `${this.baseUrl}${path}`;
 		const headers: Record<string, string> = {
+			...this.#headers,
 			Authorization: `Bearer ${key}`,
 			Accept: "application/json",
 			"Content-Type": "application/json",

package/src/providers/embeddings-server.ts

@@ -0,0 +1,151 @@
+import { type } from "@oh-my-pi/omptype";
+import * as AIError from "../error";
+import type { EmbeddingRequest, EmbeddingResult } from "../embeddings/types";
+
+export const MAX_EMBEDDINGS_BODY_BYTES = 8 * 1024 * 1024;
+
+const embeddingRequestSchema = type({
+	model: "string > 0",
+	input: "unknown",
+	"dimensions?": "unknown",
+	"encoding_format?": "unknown",
+	"user?": "unknown",
+});
+
+export class EmbeddingsWireError extends AIError.ValidationError {
+	readonly status: number;
+
+	constructor(status: number, message: string, options?: { cause?: unknown }) {
+		super(message, options);
+		this.name = "EmbeddingsWireError";
+		this.status = status;
+	}
+}
+
+export interface EmbeddingsParsedRequest {
+	modelId: string;
+	request: EmbeddingRequest;
+}
+
+function invalid(message: string): never {
+	throw new EmbeddingsWireError(400, `embeddings: ${message}`);
+}
+
+function parseInput(value: unknown): EmbeddingRequest["input"] {
+	if (typeof value === "string") {
+		if (value.length === 0) invalid("input must not be empty");
+		return value;
+	}
+	if (!Array.isArray(value) || value.length === 0) invalid("input must be a non-empty string or array");
+	if (value.every(item => typeof item === "string" && item.length > 0)) return value;
+	if (value.every(item => typeof item === "number" && Number.isFinite(item))) return value;
+	if (
+		value.every(
+			item =>
+				Array.isArray(item) &&
+				item.length > 0 &&
+				item.every(token => typeof token === "number" && Number.isFinite(token)),
+		)
+	) {
+		return value;
+	}
+	invalid("input arrays must contain non-empty strings, finite numbers, or non-empty arrays of finite numbers");
+}
+
+async function readBody(req: Request): Promise<Uint8Array> {
+	const contentLength = req.headers.get("content-length");
+	if (contentLength !== null) {
+		const declared = Number(contentLength);
+		if (Number.isFinite(declared) && declared > MAX_EMBEDDINGS_BODY_BYTES) {
+			throw new EmbeddingsWireError(413, "Request payload exceeds the 8 MiB limit");
+		}
+	}
+	const reader = req.body?.getReader();
+	if (!reader) return new Uint8Array();
+	const chunks: Uint8Array[] = [];
+	let total = 0;
+	for (;;) {
+		const { done, value } = await reader.read();
+		if (done) break;
+		total += value.byteLength;
+		if (total > MAX_EMBEDDINGS_BODY_BYTES) {
+			await reader.cancel();
+			throw new EmbeddingsWireError(413, "Request payload exceeds the 8 MiB limit");
+		}
+		chunks.push(value);
+	}
+	if (chunks.length === 1) return chunks[0]!;
+	const bytes = new Uint8Array(total);
+	let offset = 0;
+	for (const chunk of chunks) {
+		bytes.set(chunk, offset);
+		offset += chunk.byteLength;
+	}
+	return bytes;
+}
+
+/** Parse an OpenAI-compatible embeddings request with a bounded JSON body. */
+export async function parseRequest(req: Request): Promise<EmbeddingsParsedRequest> {
+	const contentType = req.headers.get("content-type")?.toLowerCase() ?? "";
+	if (!contentType.startsWith("application/json")) {
+		throw new EmbeddingsWireError(400, "Content-Type must be application/json");
+	}
+	const bytes = await readBody(req);
+	let body: unknown;
+	try {
+		body = JSON.parse(new TextDecoder().decode(bytes));
+	} catch (error) {
+		throw new EmbeddingsWireError(400, "Invalid JSON body", { cause: error });
+	}
+	const parsed = embeddingRequestSchema(body);
+	if (parsed instanceof type.errors) throw new EmbeddingsWireError(400, `embeddings: ${parsed.summary}`);
+	const dimensions = parsed.dimensions;
+	if (dimensions !== undefined && (!Number.isInteger(dimensions) || (dimensions as number) < 1)) {
+		invalid("dimensions must be a positive integer");
+	}
+	const encodingFormat = parsed.encoding_format ?? "float";
+	if (encodingFormat !== "float" && encodingFormat !== "base64") {
+		invalid('encoding_format must be "float" or "base64"');
+	}
+	if (parsed.user !== undefined && (typeof parsed.user !== "string" || parsed.user.length === 0)) {
+		invalid("user must be a non-empty string");
+	}
+	return {
+		modelId: parsed.model,
+		request: {
+			input: parseInput(parsed.input),
+			encodingFormat,
+			...(dimensions !== undefined && { dimensions: dimensions as number }),
+			...(parsed.user !== undefined && { user: parsed.user }),
+		},
+	};
+}
+
+export interface EmbeddingsResponseBody {
+	object: "list";
+	data: Array<{ object: "embedding"; index: number; embedding: number[] | string }>;
+	model: string;
+	usage: { prompt_tokens: number; total_tokens: number; cost?: number };
+}
+
+/** Encode a canonical result as an OpenAI/OpenRouter embeddings response. */
+export function encodeResponse(result: EmbeddingResult, requestedModelId: string): EmbeddingsResponseBody {
+	const billedCost = result.usage.credits?.cost;
+	return {
+		object: "list",
+		data: result.embeddings.map(item => ({ object: "embedding", ...item })),
+		model: requestedModelId,
+		usage: {
+			prompt_tokens: result.usage.input,
+			total_tokens: result.usage.totalTokens,
+			...(billedCost !== undefined && { cost: billedCost }),
+		},
+	};
+}
+
+export function formatError(status: number, errorType: string, message: string): Response {
+	return new Response(JSON.stringify({ error: { code: status, type: errorType, message } }), {
+		status,
+		headers: { "Content-Type": "application/json; charset=utf-8", "Cache-Control": "no-store" },
+	});
+}