@opengeni/sdk 1.4.0 → 1.4.4

package/src/session-proxy.ts

@@ -110,7 +110,12 @@ export type SessionProxyCreateInput = {
 export type SessionProxyMessageInput = {
   /** Absent for `create`. */
   sessionId?: string | undefined;
-  delivery: "create" | "send" | "steer" | "submit";
+  /**
+   * `realtime` is live voice: once when a call starts (refuse it, or rotate
+   * MCP credentials), and before each batch of finalized transcripts and spoken
+   * requests is saved (refuse it, or add `modelContext`).
+   */
+  delivery: "create" | "send" | "steer" | "submit" | "realtime";
 };
 
 /** Server-side additions the host attaches to one forwarded user message or response. */
@@ -118,7 +123,9 @@ export type SessionProxyMessageExtras = {
   /**
    * Model-visible context for this message (current page, time zone, today's
    * date). Placed before any context the browser sent. Not secret. Ignored for
-   * approval decisions and human-input responses, which are not new messages.
+   * approval decisions and human-input responses, which are not new messages,
+   * and when a live voice call starts. Keep it stable for the same message:
+   * a retried voice entry must carry the same context to be accepted.
    */
   modelContext?: string | undefined;
   /**
@@ -193,9 +200,10 @@ export type SessionProxyHandlerOptions = {
     | undefined;
   /**
    * Called before every forwarded user message (send, steer, composer submit,
-   * and browser-started create), approval decision, and human-input response.
-   * Return server-owned `modelContext` (messages only) and MCP credential
-   * rotations, or a `Response` to reject the action.
+   * and browser-started create), approval decision, human-input response, and
+   * live voice start and transcript save (`delivery: "realtime"`).
+   * Return server-owned `modelContext` (messages and voice transcripts only)
+   * and MCP credential rotations, or a `Response` to reject the action.
    */
   beforeForwardMessage?:
     | ((
@@ -237,6 +245,18 @@ export type SessionProxyHandlerOptions = {
    * hide the microphone. Defaults to true.
    */
   voiceInput?: boolean | undefined;
+  /**
+   * Forward live speech-to-speech voice for existing chats as the resolved
+   * user: the voice model catalog and the call lifecycle routes under
+   * `.../sessions/{id}/realtime`. Opengeni still requires that user's
+   * `sessions:control` permission, binds the call to that user and browser,
+   * runs spoken requests as ordinary steers of the chat, and meters
+   * deployment-funded voice against the organization's credits. The stock
+   * composer shows the voice button only when a voice model is available.
+   * `false` reports `realtimeVoice: false` in the client config so stock UIs
+   * hide it, and refuses the routes. Defaults to true.
+   */
+  realtimeVoice?: boolean | undefined;
   /**
    * Let the browser read a file from the session's sandbox (`POST .../fs/read`)
    * so `sandbox:` links in agent replies can be downloaded. Only `path`,
@@ -291,10 +311,20 @@ export type SessionProxyHandlerOptions = {
    * settings decide which models end users see.
    */
   modelSelection?: boolean | undefined;
-  /** SSE heartbeat interval. Defaults to 15 seconds. */
+  /**
+   * SSE heartbeat interval. Defaults to 5 seconds, under Bun.serve's default
+   * 10-second `idleTimeout`, which otherwise closes a quiet event stream.
+   */
   heartbeatMs?: number | undefined;
 };
 
+/**
+ * Default SSE heartbeat. Bun.serve closes a connection that sends nothing for
+ * 10 seconds by default, so a quiet stream (a long tool call, an idle chat)
+ * would drop and reconnect every 10 seconds through a Bun or Hono-on-Bun host.
+ */
+const SESSION_PROXY_DEFAULT_HEARTBEAT_MS = 5_000;
+
 const DEFAULT_MAX_BODY_BYTES = 1024 * 1024;
 const SEGMENT = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/;
 const QUEUE_OPERATIONS: ReadonlySet<string> = new Set(["move", "edit", "steer", "delete"]);
@@ -351,9 +381,12 @@ export function createSessionProxyHandler(
 ): (request: Request) => Promise<Response> {
   const service = isFacade(target) ? target.client : target;
   const defaultSource = isFacade(target) ? target.source : "default";
+  // One rejected-key notice per handler (each handler holds one server key).
+  const rejectedKeyNotice = { sent: false };
   const maxBodyBytes = options.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES;
   const filesEnabled = options.files ?? true;
   const voiceInputEnabled = options.voiceInput ?? true;
+  const realtimeVoiceEnabled = options.realtimeVoice ?? true;
   const sandboxFilesEnabled = options.sandboxFiles === true;
   const artifactsEnabled = options.artifacts !== undefined && options.artifacts !== false;
   const editableLiveUrl = artifactsEnabled
@@ -363,7 +396,7 @@ export function createSessionProxyHandler(
       )
     : null;
   const modelSelection = options.modelSelection ?? true;
-  const heartbeatMs = options.heartbeatMs ?? 15_000;
+  const heartbeatMs = options.heartbeatMs ?? SESSION_PROXY_DEFAULT_HEARTBEAT_MS;
   const sessionList = options.sessionList ?? "mine";
   const archiveEnabled = options.archive ?? true;
   const chats = options.chats ?? "private";
@@ -520,6 +553,39 @@ export function createSessionProxyHandler(
           ],
         };
       };
+      /**
+       * Best-effort standalone rotation before a voice call. Opengeni refuses
+       * it while a turn is running; that turn's message already refreshed them.
+       */
+      const refreshRealtimeCredentials = async (
+        sessionId: string,
+        updates: SessionMcpCredentialUpdateInput[],
+      ): Promise<void> => {
+        if (updates.length === 0) return;
+        try {
+          const servers = (await client.getSession(workspaceId, sessionId, call)).mcpServers ?? [];
+          const rotations = updates.flatMap((update) => {
+            const server = servers.find((candidate) => candidate.id === update.id);
+            return server
+              ? [
+                  {
+                    id: update.id,
+                    expectedCredentialVersion: server.credentialVersion,
+                    expectedServerUrl: server.url,
+                    headers: update.headers,
+                  },
+                ]
+              : [];
+          });
+          if (rotations.length === 0) return;
+          await client.rotateSessionMcpCredentials(workspaceId, sessionId, {
+            operationKey: crypto.randomUUID(),
+            updates: rotations,
+          });
+        } catch (error) {
+          if (!(error instanceof OpenGeniApiError)) throw error;
+        }
+      };
       /** Browser input sanitized, then server-owned extras merged in. */
       const forwardMessage = async (
         value: unknown,
@@ -617,6 +683,7 @@ export function createSessionProxyHandler(
             // Whether the stock "New chat" and "Archive" actions can succeed.
             sessionCreation: options.createSession !== undefined,
             archive: archiveEnabled,
+            ...(realtimeVoiceEnabled ? {} : { realtimeVoice: false }),
             // Explicit true also tells stock UIs to offer end users the model picker.
             ...(modelSelection
               ? options.modelSelection === true
@@ -650,6 +717,10 @@ export function createSessionProxyHandler(
       if (area === "model-catalog" && tail.length === 0 && method === "GET") {
         return await read(`${base}/model-catalog`);
       }
+      if (area === "realtime-model-catalog" && tail.length === 0 && method === "GET") {
+        if (!realtimeVoiceEnabled) return errorJson(404, "route_not_allowed", "Not found.");
+        return await read(`${base}/realtime-model-catalog`);
+      }
       if (area === "live-events" && tail.length === 1 && tail[0] === "stream" && method === "GET") {
         // Surface authorization failures as HTTP status before streaming.
         await client.requestJson("GET", base, undefined, {}, call);
@@ -879,6 +950,34 @@ export function createSessionProxyHandler(
           ? payload
           : json(await client.requestJson("POST", path, payload));
 
+      if (op[0] === "realtime") {
+        if (!realtimeVoiceEnabled) return errorJson(404, "route_not_allowed", "Not found.");
+        const realtime = realtimeRoute(method, op.slice(1));
+        if (!realtime) return errorJson(404, "route_not_allowed", "Not found.");
+        if (!body) reject(400, "invalid_body", "A JSON object body is required.");
+        const path = `${session}/${op.join("/")}`;
+        if (realtime === "begin") {
+          // The host may refuse a call; MCP credentials (including the tool
+          // server's per-user token) are refreshed before voice can delegate.
+          const extras = await messageExtras({
+            sessionId,
+            delivery: "realtime",
+          });
+          if (extras instanceof Response) return extras;
+          await refreshRealtimeCredentials(sessionId, extras?.mcpCredentialUpdates ?? []);
+        }
+        let payload: Record<string, unknown> = body;
+        if (realtime === "sync" && options.beforeForwardMessage && hasRealtimeMessages(body)) {
+          const extras = await options.beforeForwardMessage(
+            { sessionId, delivery: "realtime" },
+            context,
+          );
+          if (extras instanceof Response) return extras;
+          payload = withRealtimeContext(body, extras?.modelContext);
+        }
+        return json(await client.requestJson(method, path, payload));
+      }
+
       switch (route) {
         case "GET ":
           return await read(session);
@@ -1020,7 +1119,7 @@ export function createSessionProxyHandler(
       }
       return errorJson(404, "route_not_allowed", "Not found.");
     } catch (error) {
-      return errorResponse(error);
+      return errorResponse(error, rejectedKeyNotice);
     }
   };
 }
@@ -1033,6 +1132,67 @@ const PROXY_METHODS: readonly string[] = ["GET", "POST", "PUT", "PATCH", "DELETE
 
 const UPLOAD_FIELDS = ["scope", "filename", "contentType", "sizeBytes", "sha256"] as const;
 
+type RealtimeRoute = "begin" | "connect" | "heartbeat" | "end" | "activate" | "sync";
+const REALTIME_CONNECT_ROUTES: ReadonlySet<string> = new Set(["webrtc", "gateway", "supergrok"]);
+/** Ledger entries that become (or join) messages and so accept `modelContext`. */
+const REALTIME_MESSAGE_KINDS: ReadonlySet<string> = new Set([
+  "delegation_call",
+  "user_transcript",
+  "assistant_transcript",
+]);
+
+/**
+ * The live voice routes a browser call uses, after `realtime/`: begin, provider
+ * connect, heartbeat, end, connection activation, and transcript sync. Bodies
+ * pass to Opengeni, which validates them and binds the call to the acting user.
+ */
+function realtimeRoute(method: string, rest: string[]): RealtimeRoute | null {
+  if (rest.length === 0) return method === "POST" ? "begin" : null;
+  if (rest.length === 1) {
+    if (REALTIME_CONNECT_ROUTES.has(rest[0]!)) return method === "POST" ? "connect" : null;
+    return method === "DELETE" ? "end" : null;
+  }
+  if (rest.length === 2 && rest[1] === "heartbeat") return method === "PATCH" ? "heartbeat" : null;
+  if (rest.length === 2 && rest[1] === "sync") return method === "POST" ? "sync" : null;
+  if (rest.length === 4 && rest[1] === "connections" && rest[3] === "activate") {
+    return method === "POST" ? "activate" : null;
+  }
+  return null;
+}
+
+function hasRealtimeMessages(body: Record<string, unknown>): boolean {
+  return (
+    Array.isArray(body.entries) &&
+    body.entries.some(
+      (entry) =>
+        entry !== null &&
+        typeof entry === "object" &&
+        REALTIME_MESSAGE_KINDS.has((entry as { kind?: unknown }).kind as string),
+    )
+  );
+}
+
+/** Server context goes before the browser's on each message-bearing entry. */
+function withRealtimeContext(
+  body: Record<string, unknown>,
+  serverContext: string | undefined,
+): Record<string, unknown> {
+  if (!serverContext?.trim() || !Array.isArray(body.entries)) return body;
+  return {
+    ...body,
+    entries: body.entries.map((entry: unknown) => {
+      if (entry === null || typeof entry !== "object") return entry;
+      const record = entry as Record<string, unknown>;
+      if (!REALTIME_MESSAGE_KINDS.has(record.kind as string)) return entry;
+      const modelContext = joinContext(
+        serverContext,
+        typeof record.modelContext === "string" ? record.modelContext : undefined,
+      );
+      return modelContext ? { ...record, modelContext } : entry;
+    }),
+  };
+}
+
 type NormalizedToolServer = SessionProxyToolServer & {
   url: string;
   id: string;
@@ -1724,8 +1884,24 @@ function workspaceLiveStream(
   });
 }
 
+/**
+ * The proxy always calls Opengeni with the server's own key, so an upstream
+ * 401 means that key is wrong, expired or revoked, never that the end user is
+ * signed out. Tell the developer once, in the server log, what to fix.
+ */
+function warnRejectedApiKey(error: OpenGeniApiError, notice: { sent: boolean }): void {
+  if (notice.sent) return;
+  notice.sent = true;
+  console.warn(
+    `[@opengeni/sdk] Opengeni rejected the session proxy's API key (401${
+      error.correlationId ? `, reference ${error.correlationId}` : ""
+    }). It may be expired, revoked or mistyped: create a new key in Opengeni under ` +
+      "Organization settings > Developer and update OPENGENI_API_KEY on the server.",
+  );
+}
+
 /** Preserve OpenGeni's error envelope so the browser SDK keeps codes, retryability, and outcome facts. */
-function errorResponse(error: unknown): Response {
+function errorResponse(error: unknown, rejectedKeyNotice: { sent: boolean }): Response {
   if (error instanceof ProxyRejection) return errorJson(error.status, error.code, error.message);
   if (error instanceof OpenGeniSetupError) {
     return json(
@@ -1741,6 +1917,7 @@ function errorResponse(error: unknown): Response {
     );
   }
   if (error instanceof OpenGeniApiError) {
+    if (error.status === 401) warnRejectedApiKey(error, rejectedKeyNotice);
     const status = error.status >= 400 && error.status <= 599 ? error.status : 502;
     // A decoded upstream envelope is forwarded verbatim; the SDK only retains decodable bodies.
     if (error.body) {
@@ -1769,5 +1946,8 @@ function errorResponse(error: unknown): Response {
   if (error instanceof Error && error.name === "AbortError") {
     return errorJson(499, "aborted", "The request was aborted.");
   }
+  // The browser gets no details; the host's server log needs the cause
+  // (for example a missing OPENGENI_API_KEY).
+  console.error("[@opengeni/sdk] Session proxy request failed:", error);
   return errorJson(500, "proxy_error", "Request could not be completed.");
 }

package/src/types.ts

@@ -4301,6 +4301,13 @@ export type ClientConfig = {
   modelSelection?: boolean | undefined;
   /** Session proxy sandbox-path download opt-in; absent on native deployments. */
   sandboxFiles?: boolean | undefined;
+  /**
+   * `false` when a host's session proxy turns live voice off
+   * (`createSessionProxyHandler({ realtimeVoice: false })`), so UIs hide the
+   * voice button. Otherwise absent: availability comes from the workspace's
+   * realtime model catalog.
+   */
+  realtimeVoice?: boolean | undefined;
   /**
    * Session proxy only: the workspace the proxy resolved for this user, so a
    * browser pointed at the proxy (`<OpenGeniChat baseUrl=... />`) needs no
@@ -7435,6 +7442,12 @@ export type CapabilityRuntime = {
           | "missing_verification";
       }
     | undefined;
+  /**
+   * Present when connecting needs an operator-registered OAuth client because
+   * the provider refuses self-registration; `configured` is whether this
+   * deployment has one. Connector surfaces offer the row only when true.
+   */
+  operatorOAuthClient?: { configured: boolean } | undefined;
 };
 
 export type CapabilityCatalogItem = {
@@ -8239,6 +8252,8 @@ export type PromotionalCreditScope = {
 export type PromotionalCreditBalance = PromotionalCreditScope & {
   grantId: string;
   remainingMicros: number;
+  /** Also pays for dictation and live voice, like general credits (signup credits). */
+  coversVoice?: boolean | undefined;
 };
 
 export type BillingBalance = {