@opengeni/api-router 1.3.0 → 1.4.0

package/src/routes/workspaces.ts

@@ -1,3 +1,9 @@
+import { AZURE_LIVE_MODEL_ID } from "../azure-live";
+import {
+  createRealtimeVoiceBilling,
+  deploymentRealtimeVoice,
+  realtimeVoiceOfferProblem,
+} from "@opengeni/core";
 import { withDirectModelProviders } from "@opengeni/config";
 import { listConnectionsMetadata } from "@opengeni/db";
 import {
@@ -950,16 +956,47 @@ export function registerWorkspaceRoutes(app: Hono, deps: ApiRouteDeps): void {
         : { available: false, unavailableReason: credentialReason };
     };
     const gatewayModels = Object.values(AI_GATEWAY_REALTIME_MODELS);
-    const models = [
-      ...gatewayModels.map((model, index) => ({
+    // Deployment-funded voice is listed only when this deployment can offer
+    // it (configured, and priced while credits are enforced). Hosted GPT Live
+    // replaces the managed Gateway choices when configured.
+    const managedCandidates = [
+      {
+        id: AZURE_LIVE_MODEL_ID,
+        label: "GPT Live 1",
+        description: "Realtime voice with session delegation",
+      },
+      ...gatewayModels.map((model) => ({
         id: model.managedModelId,
         label: model.label,
-        provider: "OpenGeni" as const,
         description: model.description,
-        ...availability(
-          Boolean(deps.settings.vercelAiGatewayApiKey),
-          "Opengeni Gateway voice is not configured",
-        ),
+      })),
+    ]
+      .filter((model) => {
+        const voice = deploymentRealtimeVoice(deps.settings, model.id);
+        return voice !== null && realtimeVoiceOfferProblem(deps.settings, voice) === null;
+      })
+      .filter((model, _index, offered) =>
+        offered.some((candidate) => candidate.id === AZURE_LIVE_MODEL_ID)
+          ? model.id === AZURE_LIVE_MODEL_ID
+          : true,
+      );
+    const hasCredits =
+      managedCandidates.length > 0 &&
+      (await createRealtimeVoiceBilling({
+        db: deps.db,
+        settings: deps.settings,
+      }).hasSpendableCredits(grant.accountId));
+    const models = [
+      ...managedCandidates.map((model, index) => ({
+        ...model,
+        provider: "OpenGeni" as const,
+        ...(hasCredits
+          ? { available: true, unavailableReason: null, unavailableCode: null }
+          : {
+              available: false,
+              unavailableReason: "Add Opengeni credits to use live voice",
+              unavailableCode: "insufficient_credits",
+            }),
         recommended: index === 0,
       })),
       {

package/src/sandbox/channel-a.ts

@@ -69,6 +69,7 @@ import {
   SandboxResumeIdentityMismatchError,
   SandboxResumeIdentityUnavailableError,
   RoutingActiveRouteChangedError,
+  RoutingMutationOutputRejectedError,
   RoutingWorkspaceRootChangedError,
   SelfhostedWorkspaceRootChangedError,
   ChannelAConflictError,
@@ -1214,6 +1215,20 @@ export function mapChannelAError(error: unknown, waitSignal?: AbortSignal): unkn
     error instanceof SandboxResumeIdentityUnavailableError
   )
     return new HTTPException(409, { message: error.message });
+  if (error instanceof RoutingMutationOutputRejectedError) {
+    const revoked = error.reasonCode === "authority_revoked";
+    return new ApiHttpError(revoked ? 403 : 409, {
+      code: revoked ? "forbidden" : "conflict",
+      message: error.message,
+      retryable: false,
+      outcomeUnknown: false,
+      details: {
+        code: error.code,
+        reasonCode: error.reasonCode,
+        physicalOutcome: "resolved",
+      },
+    });
+  }
   if (
     error instanceof RoutingActiveRouteChangedError ||
     error instanceof RoutingWorkspaceRootChangedError ||
@@ -1309,6 +1324,13 @@ export function channelAOperationFailureDiagnostic(
       errorCode: "sandbox_channel_a_cancelled",
     };
   }
+  if (error instanceof RoutingMutationOutputRejectedError) {
+    return {
+      reason: "request_rejected",
+      status: error.reasonCode === "authority_revoked" ? 403 : 409,
+      errorCode: "sandbox_channel_a_operation_failed",
+    };
+  }
   if (error instanceof SandboxProviderReadLockUnavailableError) {
     return {
       reason: "provider_read_busy",

package/src/transcription/billing-refusal.ts

@@ -0,0 +1,14 @@
+import type { TranscriptionBillingRefusedError } from "@opengeni/core";
+import type { Context } from "hono";
+
+/**
+ * Same body shape as session admission refusals (`code` + `message`, plus
+ * allowance `scope`/`resetsAt`), so SDK/React clients classify it as a
+ * definitive credit/allowance refusal rather than a retryable failure.
+ */
+export function transcriptionBillingRefusal(
+  c: Context,
+  error: TranscriptionBillingRefusedError,
+): Response {
+  return c.json({ code: error.code, message: error.message, ...error.details }, error.status);
+}

package/src/transcription/normalize.ts

@@ -0,0 +1,77 @@
+import { TranscriptionServiceError } from "@opengeni/core";
+import { createFfmpegTranscriptionSegmenter, TranscriptionSegmenterError } from "./segmenter";
+import { wavDurationSeconds } from "./wav-duration";
+
+/**
+ * Decode one uploaded recording to 16 kHz mono PCM WAV with a hard decode
+ * ceiling, and measure its duration from the bytes the server produced.
+ */
+export type TranscriptionAudioNormalizer = (input: {
+  audio: Uint8Array;
+  mimeType: string;
+  /** Longest accepted recording; anything longer is refused, never truncated. */
+  maxDurationSeconds: number;
+  signal?: AbortSignal | undefined;
+}) => Promise<{ bytes: Uint8Array; durationSeconds: number }>;
+
+/**
+ * ffmpeg decodes at most `maxDurationSeconds + 1` seconds (`-t`), so a long or
+ * hostile upload costs bounded CPU and disk. Output that reaches that ceiling
+ * was longer than the limit and is refused as `too_large` before any provider
+ * call, so the transcript the user receives is never silently cut short.
+ */
+export function createFfmpegAudioNormalizer(input: {
+  ffmpegPath: string;
+}): TranscriptionAudioNormalizer {
+  const segmenter = createFfmpegTranscriptionSegmenter({ ffmpegPath: input.ffmpegPath });
+  return async ({ audio, mimeType, maxDurationSeconds, signal }) => {
+    if (!Number.isFinite(maxDurationSeconds) || maxDurationSeconds <= 0) {
+      throw new TranscriptionServiceError({
+        code: "unavailable",
+        message: "Voice input duration limit is not configured.",
+      });
+    }
+    const decodeSeconds = Math.ceil(maxDurationSeconds) + 1;
+    // One segment strictly longer than the decode ceiling: never a second file.
+    const segmentSeconds = decodeSeconds + 1;
+    let normalized: Uint8Array | undefined;
+    try {
+      for await (const segment of segmenter.segment({
+        sourceMimeType: mimeType,
+        chunks: (async function* () {
+          yield audio;
+        })(),
+        providerSegmentSeconds: segmentSeconds,
+        totalDurationMilliseconds: segmentSeconds * 1_000,
+        maxDecodeSeconds: decodeSeconds,
+        ...(signal ? { signal } : {}),
+      })) {
+        if (normalized) {
+          throw new TranscriptionServiceError({ code: "too_large", message: "Audio is too long." });
+        }
+        normalized = segment.bytes;
+      }
+    } catch (error) {
+      if (error instanceof TranscriptionSegmenterError) {
+        throw new TranscriptionServiceError({
+          code: error.code === "unknown" ? "invalid_audio" : error.code,
+          message: "Audio could not be prepared for transcription.",
+          retryable: error.retryable,
+        });
+      }
+      throw error;
+    }
+    const durationSeconds = normalized ? wavDurationSeconds(normalized) : null;
+    if (!normalized || durationSeconds === null || !(durationSeconds > 0)) {
+      throw new TranscriptionServiceError({
+        code: "invalid_audio",
+        message: "Audio could not be decoded.",
+      });
+    }
+    // Reaching the decode ceiling (within one 20 ms frame) means the source ran past it.
+    if (durationSeconds > decodeSeconds - 0.02) {
+      throw new TranscriptionServiceError({ code: "too_large", message: "Audio is too long." });
+    }
+    return { bytes: normalized, durationSeconds };
+  };
+}

package/src/transcription/providers/azure-mai.ts

@@ -0,0 +1,98 @@
+import type { VoiceInputPricing } from "@opengeni/config";
+import type { TranscriptionProvider } from "@opengeni/core";
+import { createFfmpegAudioNormalizer } from "../normalize";
+import { fetchError, responseError } from "./openai";
+
+/** Azure Speech file transcription; credentials and model selection stay server-side. */
+export function createMaiTranscriptionProvider(input: {
+  endpoint: string;
+  apiKey: string;
+  apiVersion: string;
+  model: string;
+  pricing?: VoiceInputPricing | null;
+  ffmpegPath: string;
+  fetch?: typeof fetch;
+}): TranscriptionProvider {
+  const fetchImpl = input.fetch ?? fetch;
+  const normalize = createFfmpegAudioNormalizer({ ffmpegPath: input.ffmpegPath });
+  return {
+    id: "azure-mai",
+    supportsServerDeadline: true,
+    deploymentFunded: { model: input.model, pricing: input.pricing ?? null },
+    available: () => Boolean(input.apiKey),
+    async transcribe({ audio, mimeType, filename, requestId, signal }) {
+      let bytes = audio;
+      let contentType = mimeType;
+      let name = filename;
+      // Resumable segments and credit-billed one-shot uploads already arrive
+      // as server-normalized WAV. Anything else (credit billing off) gets the
+      // same bounded decoder before entering Speech's file API.
+      if (!["audio/wav", "audio/x-wav", "audio/mpeg", "audio/flac"].includes(mimeType)) {
+        const normalized = await normalize({
+          audio,
+          mimeType,
+          maxDurationSeconds: 600,
+          ...(signal ? { signal } : {}),
+        });
+        bytes = normalized.bytes;
+        contentType = "audio/wav";
+        name = "audio.wav";
+      }
+      const form = new FormData();
+      form.append("audio", new Blob([Uint8Array.from(bytes).buffer], { type: contentType }), name);
+      form.append(
+        "definition",
+        JSON.stringify({ enhancedMode: { enabled: true, model: input.model } }),
+      );
+      let response: Response;
+      try {
+        response = await fetchImpl(
+          `${input.endpoint.replace(/\/+$/, "")}/speechtotext/transcriptions:transcribe?api-version=${encodeURIComponent(input.apiVersion)}`,
+          {
+            method: "POST",
+            headers: {
+              "Ocp-Apim-Subscription-Key": input.apiKey,
+              "x-opengeni-request-id": requestId,
+            },
+            body: form,
+            ...(signal ? { signal } : {}),
+          },
+        );
+      } catch (error) {
+        throw fetchError(error);
+      }
+      if (!response.ok) throw responseError(response.status);
+      const body = await response.json().catch(() => null);
+      if (
+        !body ||
+        !Array.isArray(body.combinedPhrases) ||
+        !body.combinedPhrases.every(
+          (phrase: unknown) =>
+            typeof phrase === "object" &&
+            phrase !== null &&
+            "text" in phrase &&
+            typeof phrase.text === "string",
+        )
+      ) {
+        throw responseError(502);
+      }
+      return {
+        usage:
+          typeof body.durationMilliseconds === "number" &&
+          Number.isFinite(body.durationMilliseconds) &&
+          body.durationMilliseconds >= 0
+            ? { kind: "duration" as const, seconds: body.durationMilliseconds / 1000 }
+            : null,
+        text: body.combinedPhrases.map((phrase: { text: string }) => phrase.text).join("\n"),
+        languages: [
+          ...new Set<string>(
+            (Array.isArray(body.phrases) ? body.phrases : []).flatMap(
+              (phrase: { locale?: unknown }) =>
+                typeof phrase?.locale === "string" && phrase.locale ? [phrase.locale] : [],
+            ),
+          ),
+        ],
+      };
+    },
+  };
+}

package/src/transcription/providers/azure-openai.ts

@@ -1,5 +1,6 @@
+import { voiceInputModelReportsDurationOnly, type VoiceInputPricing } from "@opengeni/config";
 import type { TranscriptionProvider } from "@opengeni/core";
-import { fetchError, responseError } from "./openai";
+import { fetchError, parseTranscriptionResponseBody, responseError } from "./openai";
 
 export function createAzureOpenAiTranscriptionProvider(input: {
   endpoint: string;
@@ -7,17 +8,27 @@ export function createAzureOpenAiTranscriptionProvider(input: {
   apiVersion: string;
   apiKey: string | null;
   adToken: string | null;
+  /** Underlying model of the deployment; defaults to the deployment name. */
+  model?: string;
+  pricing?: VoiceInputPricing | null;
   fetch?: typeof fetch;
 }): TranscriptionProvider {
   const fetchImpl = input.fetch ?? fetch;
-  const url = `${input.endpoint}/openai/deployments/${encodeURIComponent(input.deployment)}/audio/transcriptions?api-version=${encodeURIComponent(input.apiVersion)}`;
+  const model = input.model ?? input.deployment;
+  const endpoint = input.endpoint.replace(/\/+$/, "");
+  const url = `${endpoint}/openai/deployments/${encodeURIComponent(input.deployment)}/audio/transcriptions?api-version=${encodeURIComponent(input.apiVersion)}`;
   return {
     id: "azure-openai",
     supportsServerDeadline: true,
+    deploymentFunded: { model, pricing: input.pricing ?? null },
     available: () => Boolean(input.apiKey || input.adToken),
     async transcribe({ audio, mimeType, filename, requestId, signal }) {
       const form = new FormData();
       form.append("file", new Blob([Uint8Array.from(audio).buffer], { type: mimeType }), filename);
+      // Whisper reports its billed audio duration only in verbose_json.
+      if (voiceInputModelReportsDurationOnly(model)) {
+        form.append("response_format", "verbose_json");
+      }
       const headers: Record<string, string> = input.apiKey
         ? { "api-key": input.apiKey, "x-opengeni-request-id": requestId }
         : { Authorization: `Bearer ${input.adToken}`, "x-opengeni-request-id": requestId };
@@ -34,13 +45,11 @@ export function createAzureOpenAiTranscriptionProvider(input: {
       }
       if (!response.ok) throw responseError(response.status);
       const body = await response.json().catch(() => null);
-      if (!body || typeof body.text !== "string") {
+      const parsed = parseTranscriptionResponseBody(body);
+      if (!parsed) {
         throw responseError(502);
       }
-      return {
-        text: body.text,
-        languages: typeof body.language === "string" && body.language ? [body.language] : [],
-      };
+      return parsed;
     },
   };
 }

package/src/transcription/providers/openai.ts

@@ -1,20 +1,31 @@
+import {
+  voiceInputModelReportsDurationOnly,
+  type VoiceInputPricing,
+  type VoiceInputUsage,
+} from "@opengeni/config";
 import { type TranscriptionProvider, TranscriptionServiceError } from "@opengeni/core";
 
 export function createOpenAiTranscriptionProvider(input: {
   apiKey: string;
   baseUrl: string;
   model: string;
+  pricing?: VoiceInputPricing | null;
   fetch?: typeof fetch;
 }): TranscriptionProvider {
   const fetchImpl = input.fetch ?? fetch;
   return {
     id: "openai",
     supportsServerDeadline: true,
+    deploymentFunded: { model: input.model, pricing: input.pricing ?? null },
     available: () => true,
     async transcribe({ audio, mimeType, filename, requestId, signal }) {
       const form = new FormData();
       form.append("file", audioBlob(audio, mimeType), filename);
       form.append("model", input.model);
+      // Whisper reports its billed audio duration only in verbose_json.
+      if (voiceInputModelReportsDurationOnly(input.model)) {
+        form.append("response_format", "verbose_json");
+      }
       let response: Response;
       try {
         response = await fetchImpl(`${input.baseUrl}/audio/transcriptions`, {
@@ -32,20 +43,83 @@ export function createOpenAiTranscriptionProvider(input: {
       }
       if (!response.ok) throw responseError(response.status);
       const body = await response.json().catch(() => null);
-      if (!body || typeof body.text !== "string") {
+      const parsed = parseTranscriptionResponseBody(body);
+      if (!parsed) {
         throw new TranscriptionServiceError({
           code: "provider",
           message: "Invalid transcription response.",
         });
       }
-      return {
-        text: body.text,
-        languages: typeof body.language === "string" && body.language ? [body.language] : [],
-      };
+      return parsed;
     },
   };
 }
 
+/**
+ * OpenAI-compatible `/audio/transcriptions` JSON (OpenAI and Azure OpenAI).
+ * `usage` is `{type:"tokens", input_tokens, input_token_details:{audio_tokens,
+ * text_tokens}, output_tokens, total_tokens}` for the gpt-4o-transcribe family
+ * or `{type:"duration", seconds}` for duration-billed models; Whisper
+ * `verbose_json` carries the processed audio `duration`. Unusable usage is
+ * dropped (null), never guessed.
+ */
+export function parseTranscriptionResponseBody(
+  body: unknown,
+): { text: string; languages: string[]; usage: VoiceInputUsage | null } | null {
+  if (!body || typeof body !== "object") return null;
+  const record = body as Record<string, unknown>;
+  if (typeof record.text !== "string") return null;
+  const languages =
+    typeof record.language === "string" && record.language
+      ? [record.language]
+      : Array.isArray(record.languages)
+        ? record.languages.flatMap((language) => {
+            const code = typeof language === "string" ? language : language?.code;
+            return typeof code === "string" && code ? [code] : [];
+          })
+        : [];
+  return { text: record.text, languages, usage: parseTranscriptionUsage(record) };
+}
+
+export function parseTranscriptionUsage(body: Record<string, unknown>): VoiceInputUsage | null {
+  const usage = body.usage;
+  if (usage && typeof usage === "object") {
+    const record = usage as Record<string, unknown>;
+    if (record.type === "tokens") {
+      const inputTokens = count(record.input_tokens);
+      const outputTokens = count(record.output_tokens);
+      if (inputTokens !== null && outputTokens !== null) {
+        const details =
+          record.input_token_details && typeof record.input_token_details === "object"
+            ? (record.input_token_details as Record<string, unknown>)
+            : {};
+        return {
+          kind: "tokens",
+          inputTokens,
+          audioInputTokens: count(details.audio_tokens) ?? 0,
+          textInputTokens: count(details.text_tokens) ?? 0,
+          outputTokens,
+        };
+      }
+    }
+    if (record.type === "duration" && seconds(record.seconds) !== null) {
+      return { kind: "duration", seconds: seconds(record.seconds)! };
+    }
+  }
+  const duration = seconds(body.duration);
+  return duration === null ? null : { kind: "duration", seconds: duration };
+}
+
+function count(value: unknown): number | null {
+  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
+}
+
+function seconds(value: unknown): number | null {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0 && value < 86_400
+    ? value
+    : null;
+}
+
 function audioBlob(audio: Uint8Array, mimeType: string): Blob {
   return new Blob([Uint8Array.from(audio).buffer], { type: mimeType });
 }

package/src/transcription/segmenter.ts

@@ -41,7 +41,9 @@ export function createFfmpegTranscriptionSegmenter(input: {
         !Number.isSafeInteger(request.providerSegmentSeconds) ||
         request.providerSegmentSeconds <= 0 ||
         !Number.isSafeInteger(request.totalDurationMilliseconds) ||
-        request.totalDurationMilliseconds <= 0
+        request.totalDurationMilliseconds <= 0 ||
+        (request.maxDecodeSeconds !== undefined &&
+          (!Number.isSafeInteger(request.maxDecodeSeconds) || request.maxDecodeSeconds <= 0))
       ) {
         throw new TranscriptionSegmenterError(
           "Audio segmentation bounds are invalid",
@@ -77,6 +79,10 @@ export function createFfmpegTranscriptionSegmenter(input: {
             "-y",
             "-i",
             inputPath,
+            // Output-side ceiling: audio past it is never decoded or written.
+            ...(request.maxDecodeSeconds !== undefined
+              ? ["-t", String(request.maxDecodeSeconds)]
+              : []),
             "-map",
             "0:a:0",
             "-vn",