@oh-my-pi/pi-ai 18.2.7 → 18.2.8

package/src/transcription/openai-transcriptions.ts

@@ -0,0 +1,133 @@
+import { calculateCost } from "@oh-my-pi/pi-catalog/models";
+import type { Api, FetchImpl, Model, Usage } from "@oh-my-pi/pi-catalog/types";
+import { type } from "@oh-my-pi/omptype";
+import { type ApiKey, withAuth } from "../auth-retry";
+import * as AIError from "../error";
+import type { TranscriptionRequest, TranscriptionResult, TranscriptionSegment, TranscriptionWord } from "./types";
+
+export interface TranscriptionOptions {
+	apiKey: ApiKey;
+	fetch?: FetchImpl;
+	signal?: AbortSignal;
+}
+
+/** Non-2xx response from an OpenAI-compatible transcription endpoint. */
+export class TranscriptionApiError extends AIError.ProviderHttpError {
+	override readonly name = "TranscriptionApiError";
+}
+
+const upstreamResponseSchema = type({
+	text: "string",
+	"language?": "string",
+	"duration?": "number",
+	"segments?": "object[]",
+	"words?": "object[]",
+	"usage?": "object",
+});
+
+interface UpstreamUsage {
+	input_tokens?: unknown;
+	output_tokens?: unknown;
+	total_tokens?: unknown;
+	cost?: unknown;
+	seconds?: unknown;
+}
+
+function finiteNumber(value: unknown): number | undefined {
+	return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined;
+}
+
+function decodeUsage(model: Model<Api>, raw: unknown): { usage: Usage; seconds?: number } {
+	const upstream = raw && typeof raw === "object" ? (raw as UpstreamUsage) : {};
+	const input = finiteNumber(upstream.input_tokens) ?? 0;
+	const output = finiteNumber(upstream.output_tokens) ?? 0;
+	const reportedTotal = finiteNumber(upstream.total_tokens);
+	const reportedCost = finiteNumber(upstream.cost);
+	const usage: Usage = {
+		input,
+		output,
+		cacheRead: 0,
+		cacheWrite: 0,
+		totalTokens: reportedTotal ?? input + output,
+		cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: reportedCost ?? 0 },
+	};
+	if (reportedCost === undefined) calculateCost(model, usage);
+	return { usage, seconds: finiteNumber(upstream.seconds) };
+}
+
+async function responseError(response: Response, model: Model<Api>): Promise<TranscriptionApiError> {
+	const text = await response.text();
+	let detail = text;
+	let code: string | undefined;
+	try {
+		const parsed: unknown = JSON.parse(text);
+		if (parsed && typeof parsed === "object" && "error" in parsed) {
+			const error = parsed.error;
+			if (error && typeof error === "object") {
+				const envelope = error as { message?: unknown; code?: unknown; type?: unknown };
+				if (typeof envelope.message === "string") detail = envelope.message;
+				if (typeof envelope.code === "string") code = envelope.code;
+				else if (typeof envelope.type === "string") code = envelope.type;
+			}
+		}
+	} catch {}
+	return new TranscriptionApiError(
+		`${model.provider}/${model.id} transcription API error (${response.status}): ${detail || response.statusText}`,
+		response.status,
+		{ headers: response.headers, code },
+	);
+}
+
+/** Call an OpenAI/OpenRouter-compatible multipart transcription endpoint. */
+export async function transcribeOpenAI(
+	model: Model<Api>,
+	request: TranscriptionRequest,
+	options: TranscriptionOptions,
+): Promise<TranscriptionResult> {
+	const form = new FormData();
+	const fileName = request.fileName?.trim() || "audio";
+	form.append("file", new File([request.audio], fileName, { type: request.mimeType }));
+	form.append("model", model.id);
+	form.append("response_format", request.responseFormat);
+	if (request.language !== undefined) form.append("language", request.language);
+	if (request.prompt !== undefined) form.append("prompt", request.prompt);
+	if (request.temperature !== undefined) form.append("temperature", String(request.temperature));
+	for (const granularity of request.timestampGranularities ?? []) {
+		form.append("timestamp_granularities[]", granularity);
+	}
+
+	const fetchImpl = options.fetch ?? fetch;
+	const response = await withAuth(
+		options.apiKey,
+		async key => {
+			const attempt = await fetchImpl(`${model.baseUrl.replace(/\/+$/, "")}/audio/transcriptions`, {
+				method: "POST",
+				headers: { Authorization: `Bearer ${key}`, Accept: "application/json" },
+				body: form,
+				signal: options.signal,
+			});
+			if (!attempt.ok) throw await responseError(attempt, model);
+			return attempt;
+		},
+		{ signal: options.signal },
+	);
+
+	const body: unknown = await response.json();
+	const parsed = upstreamResponseSchema(body);
+	if (parsed instanceof type.errors) {
+		throw new AIError.ProviderResponseError(
+			`${model.provider}/${model.id} transcription response is malformed: ${parsed.summary}`,
+			{ provider: model.provider, kind: "envelope" },
+		);
+	}
+	const decoded = decodeUsage(model, parsed.usage);
+	return {
+		text: parsed.text,
+		...(parsed.language !== undefined && { language: parsed.language }),
+		...(parsed.duration !== undefined && { duration: parsed.duration }),
+		...(parsed.segments !== undefined && { segments: parsed.segments as TranscriptionSegment[] }),
+		...(parsed.words !== undefined && { words: parsed.words as TranscriptionWord[] }),
+		...(decoded.seconds !== undefined && { seconds: decoded.seconds }),
+		usage: decoded.usage,
+	};
+}

package/src/transcription/types.ts

@@ -0,0 +1,46 @@
+import type { Usage } from "@oh-my-pi/pi-catalog/types";
+
+export type TranscriptionResponseFormat = "json" | "verbose_json";
+export type TranscriptionTimestampGranularity = "word" | "segment";
+
+export interface TranscriptionRequest {
+	audio: Uint8Array;
+	mimeType: string;
+	fileName?: string;
+	language?: string;
+	prompt?: string;
+	temperature?: number;
+	responseFormat: TranscriptionResponseFormat;
+	timestampGranularities?: TranscriptionTimestampGranularity[];
+}
+
+/** Provider-specific segment fields are preserved alongside the normalized timestamps and text. */
+export interface TranscriptionSegment {
+	id?: number | string;
+	start: number;
+	end: number;
+	text: string;
+	speaker?: number | string;
+	[key: string]: unknown;
+}
+
+/** Provider-specific word fields are preserved alongside the normalized timestamps and spelling. */
+export interface TranscriptionWord {
+	word: string;
+	start: number;
+	end: number;
+	speaker?: number | string;
+	confidence?: number;
+	[key: string]: unknown;
+}
+
+export interface TranscriptionResult {
+	text: string;
+	language?: string;
+	duration?: number;
+	segments?: TranscriptionSegment[];
+	words?: TranscriptionWord[];
+	/** Provider-billed audio duration, which may be present without verbose output. */
+	seconds?: number;
+	usage: Usage;
+}

package/src/video/index.ts

@@ -0,0 +1,34 @@
+import type { Api, Model } from "@oh-my-pi/pi-catalog/types";
+import * as AIError from "../error";
+import {
+	downloadOpenRouterVideo,
+	pollOpenRouterVideo,
+	submitOpenRouterVideo,
+	type VideoOptions,
+} from "./openrouter-video";
+import type { VideoContent, VideoGenerationRequest, VideoJob } from "./types";
+
+export * from "./openrouter-video";
+export * from "./types";
+
+/** Submit an asynchronous video generation job through the model's transport. */
+export function submitVideo(
+	model: Model<Api>,
+	request: VideoGenerationRequest,
+	options: VideoOptions,
+): Promise<VideoJob> {
+	if (model.api === "openrouter-video") return submitOpenRouterVideo(model, request, options);
+	throw new AIError.ConfigurationError(`Unsupported video API: ${model.api}`);
+}
+
+/** Poll an asynchronous video generation job through the model's transport. */
+export function pollVideo(model: Model<Api>, jobId: string, options: VideoOptions): Promise<VideoJob> {
+	if (model.api === "openrouter-video") return pollOpenRouterVideo(model, jobId, options);
+	throw new AIError.ConfigurationError(`Unsupported video API: ${model.api}`);
+}
+
+/** Stream generated video content through the model's transport. */
+export function downloadVideo(model: Model<Api>, jobId: string, options: VideoOptions): Promise<VideoContent> {
+	if (model.api === "openrouter-video") return downloadOpenRouterVideo(model, jobId, options);
+	throw new AIError.ConfigurationError(`Unsupported video API: ${model.api}`);
+}

package/src/video/openrouter-video.ts

@@ -0,0 +1,210 @@
+import type { Api, FetchImpl, Model, Usage } from "@oh-my-pi/pi-catalog/types";
+import { type } from "@oh-my-pi/omptype";
+import { type ApiKey, withAuth } from "../auth-retry";
+import * as AIError from "../error";
+import type {
+	VideoAudioReference,
+	VideoContent,
+	VideoFrameImage,
+	VideoGenerationRequest,
+	VideoImageReference,
+	VideoInputReference,
+	VideoJob,
+	VideoVideoReference,
+} from "./types";
+
+export interface VideoOptions {
+	apiKey: ApiKey;
+	fetch?: FetchImpl;
+	signal?: AbortSignal;
+}
+
+/** Non-2xx response from an OpenRouter video endpoint. */
+export class VideoApiError extends AIError.ProviderHttpError {
+	override readonly name = "VideoApiError";
+}
+
+const videoJobSchema = type({
+	id: "string > 0",
+	status: "'queued' | 'processing' | 'pending' | 'in_progress' | 'completed' | 'failed' | 'cancelled' | 'expired'",
+	"polling_url?": "string",
+	"generation_id?": "string",
+	"unsigned_urls?": "string[]",
+	"error?": "string",
+	"usage?": "object",
+});
+
+function videoBaseUrl(model: Model<Api>): string {
+	return model.baseUrl.replace(/\/+$/, "");
+}
+
+async function responseError(response: Response, model: Model<Api>): Promise<VideoApiError> {
+	const text = await response.text();
+	let detail = text;
+	let code: string | undefined;
+	try {
+		const parsed: unknown = JSON.parse(text);
+		if (parsed && typeof parsed === "object" && "error" in parsed) {
+			const error = parsed.error;
+			if (typeof error === "string") detail = error;
+			else if (error && typeof error === "object") {
+				const envelope = error as { message?: unknown; code?: unknown; type?: unknown };
+				if (typeof envelope.message === "string") detail = envelope.message;
+				if (typeof envelope.code === "string") code = envelope.code;
+				else if (typeof envelope.type === "string") code = envelope.type;
+			}
+		}
+	} catch {}
+	return new VideoApiError(
+		`${model.provider}/${model.id} video API error (${response.status}): ${detail || response.statusText}`,
+		response.status,
+		{ headers: response.headers, code },
+	);
+}
+
+function imageReference(reference: VideoImageReference): Record<string, unknown> {
+	return { type: reference.type, image_url: reference.imageUrl };
+}
+
+function frameImage(reference: VideoFrameImage): Record<string, unknown> {
+	return { ...imageReference(reference), frame_type: reference.frameType };
+}
+
+function inputReference(reference: VideoInputReference): Record<string, unknown> {
+	switch (reference.type) {
+		case "image_url":
+			return imageReference(reference);
+		case "audio_url":
+			return { type: reference.type, audio_url: (reference as VideoAudioReference).audioUrl };
+		case "video_url":
+			return { type: reference.type, video_url: (reference as VideoVideoReference).videoUrl };
+	}
+}
+
+function encodeRequest(model: Model<Api>, request: VideoGenerationRequest): Record<string, unknown> {
+	return {
+		model: model.requestModelId ?? model.id,
+		...(request.prompt !== undefined && { prompt: request.prompt }),
+		...(request.duration !== undefined && { duration: request.duration }),
+		...(request.resolution !== undefined && { resolution: request.resolution }),
+		...(request.aspectRatio !== undefined && { aspect_ratio: request.aspectRatio }),
+		...(request.size !== undefined && { size: request.size }),
+		...(request.frameImages !== undefined && { frame_images: request.frameImages.map(frameImage) }),
+		...(request.inputReferences !== undefined && {
+			input_references: request.inputReferences.map(inputReference),
+		}),
+		...(request.generateAudio !== undefined && { generate_audio: request.generateAudio }),
+		...(request.seed !== undefined && { seed: request.seed }),
+		...(request.callbackUrl !== undefined && { callback_url: request.callbackUrl }),
+		...(request.provider !== undefined && { provider: request.provider }),
+		...(request.previousJobId !== undefined && { previous_job_id: request.previousJobId }),
+		...(request.sessionId !== undefined && { session_id: request.sessionId }),
+		...(request.trace !== undefined && { trace: request.trace }),
+		...(request.user !== undefined && { user: request.user }),
+		...(request.creativity !== undefined && { creativity: request.creativity }),
+		...(request.upscaleFactor !== undefined && { upscale_factor: request.upscaleFactor }),
+	};
+}
+
+function decodeUsage(raw: unknown): Usage | undefined {
+	if (!raw || typeof raw !== "object") return undefined;
+	const cost = "cost" in raw && typeof raw.cost === "number" && Number.isFinite(raw.cost) ? raw.cost : undefined;
+	if (cost === undefined) return undefined;
+	return {
+		input: 0,
+		output: 0,
+		cacheRead: 0,
+		cacheWrite: 0,
+		totalTokens: 0,
+		cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: cost },
+	};
+}
+
+async function decodeJob(response: Response, model: Model<Api>): Promise<VideoJob> {
+	const body: unknown = await response.json();
+	const parsed = videoJobSchema(body);
+	if (parsed instanceof type.errors) {
+		throw new AIError.ProviderResponseError(
+			`${model.provider}/${model.id} video response is malformed: ${parsed.summary}`,
+			{ provider: model.provider, kind: "envelope" },
+		);
+	}
+	const usage = decodeUsage(parsed.usage);
+	return {
+		id: parsed.id,
+		status: parsed.status,
+		...(parsed.polling_url !== undefined && { pollingUrl: parsed.polling_url }),
+		...(parsed.generation_id !== undefined && { generationId: parsed.generation_id }),
+		...(parsed.unsigned_urls !== undefined && { contentUrls: parsed.unsigned_urls }),
+		...(parsed.error !== undefined && { error: parsed.error }),
+		...(usage !== undefined && { usage }),
+	};
+}
+
+async function authenticatedFetch(
+	model: Model<Api>,
+	path: string,
+	options: VideoOptions,
+	init?: Omit<RequestInit, "signal">,
+): Promise<Response> {
+	const fetchImpl = options.fetch ?? fetch;
+	return withAuth(
+		options.apiKey,
+		async key => {
+			const response = await fetchImpl(`${videoBaseUrl(model)}${path}`, {
+				...init,
+				headers: { Authorization: `Bearer ${key}`, Accept: "application/json", ...init?.headers },
+				signal: options.signal,
+			});
+			if (!response.ok) throw await responseError(response, model);
+			return response;
+		},
+		{ signal: options.signal },
+	);
+}
+
+/** Submit an asynchronous OpenRouter video generation job. */
+export async function submitOpenRouterVideo(
+	model: Model<Api>,
+	request: VideoGenerationRequest,
+	options: VideoOptions,
+): Promise<VideoJob> {
+	const response = await authenticatedFetch(model, "/videos", options, {
+		method: "POST",
+		headers: { "Content-Type": "application/json" },
+		body: JSON.stringify(encodeRequest(model, request)),
+	});
+	return decodeJob(response, model);
+}
+
+/** Poll an asynchronous OpenRouter video generation job. */
+export async function pollOpenRouterVideo(model: Model<Api>, jobId: string, options: VideoOptions): Promise<VideoJob> {
+	const response = await authenticatedFetch(model, `/videos/${encodeURIComponent(jobId)}`, options);
+	return decodeJob(response, model);
+}
+
+/** Stream generated video bytes from OpenRouter without buffering them in memory. */
+export async function downloadOpenRouterVideo(
+	model: Model<Api>,
+	jobId: string,
+	options: VideoOptions,
+): Promise<VideoContent> {
+	const response = await authenticatedFetch(model, `/videos/${encodeURIComponent(jobId)}/content`, options, {
+		headers: { Accept: "video/*" },
+	});
+	if (!response.body) {
+		throw new AIError.ProviderResponseError(`${model.provider}/${model.id} video content response has no body`, {
+			provider: model.provider,
+			kind: "envelope",
+		});
+	}
+	const contentLengthHeader = response.headers.get("content-length");
+	const contentLength = contentLengthHeader === null ? undefined : Number(contentLengthHeader);
+	return {
+		body: response.body,
+		contentType: response.headers.get("content-type") ?? "application/octet-stream",
+		...(contentLength !== undefined && Number.isSafeInteger(contentLength) && contentLength >= 0
+			? { contentLength }
+			: {}),
+	};
+}

package/src/video/types.ts

@@ -0,0 +1,72 @@
+import type { Usage } from "@oh-my-pi/pi-catalog/types";
+
+export type VideoResolution = "360p" | "480p" | "720p" | "768p" | "1080p" | "1K" | "2K" | "4K";
+export type VideoAspectRatio = "16:9" | "9:16" | "1:1" | "4:3" | "3:4" | "3:2" | "2:3" | "21:9" | "9:21";
+
+export interface VideoImageReference {
+	type: "image_url";
+	imageUrl: { url: string };
+}
+
+export interface VideoFrameImage extends VideoImageReference {
+	frameType: "first_frame" | "last_frame";
+}
+
+export interface VideoAudioReference {
+	type: "audio_url";
+	audioUrl: { url: string };
+}
+
+export interface VideoVideoReference {
+	type: "video_url";
+	videoUrl: { url: string };
+}
+
+export type VideoInputReference = VideoImageReference | VideoAudioReference | VideoVideoReference;
+
+/** Canonical camel-case form of OpenRouter's asynchronous video submit request. */
+export interface VideoGenerationRequest {
+	prompt?: string;
+	duration?: number;
+	resolution?: VideoResolution;
+	aspectRatio?: VideoAspectRatio;
+	size?: string;
+	frameImages?: VideoFrameImage[];
+	inputReferences?: VideoInputReference[];
+	generateAudio?: boolean;
+	seed?: number;
+	callbackUrl?: string;
+	provider?: { options?: Record<string, unknown> };
+	previousJobId?: string;
+	sessionId?: string;
+	trace?: Record<string, unknown>;
+	user?: string;
+	creativity?: number;
+	upscaleFactor?: number;
+}
+
+export type VideoJobStatus =
+	| "queued"
+	| "processing"
+	| "pending"
+	| "in_progress"
+	| "completed"
+	| "failed"
+	| "cancelled"
+	| "expired";
+
+export interface VideoJob {
+	id: string;
+	status: VideoJobStatus;
+	pollingUrl?: string;
+	generationId?: string;
+	contentUrls?: string[];
+	error?: string;
+	usage?: Usage;
+}
+
+export interface VideoContent {
+	body: ReadableStream<Uint8Array>;
+	contentType: string;
+	contentLength?: number;
+}