@opengeni/sdk 1.3.0 → 1.4.2

package/src/realtime.ts

@@ -1,3 +1,4 @@
+import { createAzureLiveTransportStarter } from "./azure-live-transport";
 /**
  * Public, provider-neutral realtime browser SDK.
  *
@@ -34,7 +35,7 @@ export type SessionRealtimeClientLike = CodexRealtimeControllerClient & {
   ): Promise<WorkspaceRealtimeModelCatalogResponse>;
 };
 
-export type SessionRealtimeTransportKind = "codex" | "gateway" | "xai-subscription";
+export type SessionRealtimeTransportKind = "codex" | "gateway" | "xai-subscription" | "azure-live";
 
 /** Provider-neutral names for the existing, battle-tested controller projection. */
 export type SessionRealtimeController = CodexRealtimeController;
@@ -57,6 +58,7 @@ export type CreateSessionRealtimeControllerOptions = Omit<
 export function sessionRealtimeTransportKind(
   model: SessionRealtimeModel,
 ): SessionRealtimeTransportKind {
+  if (model === "opengeni-azure/gpt-live-1") return "azure-live";
   if (model === "gpt-live-1-boulder-alpha") return "codex";
   if (model === "supergrok/grok-voice-think-fast-2.0") return "xai-subscription";
   return "gateway";
@@ -74,11 +76,18 @@ export function createSessionRealtimeController(
   return createCodexRealtimeController({
     ...options,
     ownerStorageNamespace: sessionRealtimeOwnerStorageNamespace(options.model),
-    ...(transport === "gateway"
-      ? { startTransport: createGatewayRealtimeTransportStarter() }
-      : transport === "xai-subscription"
-        ? { startTransport: createXaiSubscriptionRealtimeTransportStarter() }
-        : {}),
+    ...(transport === "azure-live"
+      ? {
+          startTransport: createAzureLiveTransportStarter({
+            remoteAudio: options.remoteAudio,
+            createPeerConnection: options.createPeerConnection,
+          }),
+        }
+      : transport === "gateway"
+        ? { startTransport: createGatewayRealtimeTransportStarter() }
+        : transport === "xai-subscription"
+          ? { startTransport: createXaiSubscriptionRealtimeTransportStarter() }
+          : {}),
   });
 }
 

package/src/session-proxy-client.ts

@@ -31,15 +31,19 @@ export function getSessionProxyWorkspaceGrant(
   );
 }
 
-/** Exact association read. The API reauthorizes the source session on every call. */
+/**
+ * Exact association read. The API reauthorizes the source session on every
+ * call. `retained` proves a generated image or video, or a published sandbox
+ * file, originated in that session.
+ */
 export function getSessionProxyArtifactAssociation(
   client: Pick<OpenGeniEmbeddingClient, "requestJson">,
   workspaceId: string,
   sessionId: string,
-  kind: "editable" | "site",
+  kind: "editable" | "site" | "retained",
   artifactId: string,
   options: OpenGeniRequestOptions = {},
-): Promise<{ sessionId: string; artifactId: string; kind: "editable" | "site" }> {
+): Promise<{ sessionId: string; artifactId: string; kind: "editable" | "site" | "retained" }> {
   return client.requestJson(
     "GET",
     `/v1/workspaces/${encodeURIComponent(workspaceId)}/sessions/${encodeURIComponent(sessionId)}/artifact-associations/${encodeURIComponent(artifactId)}`,

package/src/session-proxy.ts

@@ -10,6 +10,7 @@ import type {
   FileResourceRef,
   LatencyMode,
   ReasoningEffort,
+  RetainedArtifactContent,
   SessionMcpCredentialUpdateInput,
 } from "./types";
 import { mintToolToken, resolveToolTokenSecret } from "./tool-auth";
@@ -109,7 +110,12 @@ export type SessionProxyCreateInput = {
 export type SessionProxyMessageInput = {
   /** Absent for `create`. */
   sessionId?: string | undefined;
-  delivery: "create" | "send" | "steer" | "submit";
+  /**
+   * `realtime` is live voice: once when a call starts (refuse it, or rotate
+   * MCP credentials), and before each batch of finalized transcripts and spoken
+   * requests is saved (refuse it, or add `modelContext`).
+   */
+  delivery: "create" | "send" | "steer" | "submit" | "realtime";
 };
 
 /** Server-side additions the host attaches to one forwarded user message or response. */
@@ -117,7 +123,9 @@ export type SessionProxyMessageExtras = {
   /**
    * Model-visible context for this message (current page, time zone, today's
    * date). Placed before any context the browser sent. Not secret. Ignored for
-   * approval decisions and human-input responses, which are not new messages.
+   * approval decisions and human-input responses, which are not new messages,
+   * and when a live voice call starts. Keep it stable for the same message:
+   * a retried voice entry must carry the same context to be accepted.
    */
   modelContext?: string | undefined;
   /**
@@ -192,9 +200,10 @@ export type SessionProxyHandlerOptions = {
     | undefined;
   /**
    * Called before every forwarded user message (send, steer, composer submit,
-   * and browser-started create), approval decision, and human-input response.
-   * Return server-owned `modelContext` (messages only) and MCP credential
-   * rotations, or a `Response` to reject the action.
+   * and browser-started create), approval decision, human-input response, and
+   * live voice start and transcript save (`delivery: "realtime"`).
+   * Return server-owned `modelContext` (messages and voice transcripts only)
+   * and MCP credential rotations, or a `Response` to reject the action.
    */
   beforeForwardMessage?:
     | ((
@@ -218,7 +227,15 @@ export type SessionProxyHandlerOptions = {
   basePath?: string | undefined;
   /** Maximum JSON request body. Defaults to 1 MiB. */
   maxBodyBytes?: number | undefined;
-  /** Expose composer file attachments (upload begin/complete, download URL). Defaults to true. */
+  /**
+   * Expose composer file attachments (upload begin/complete, download URL) and
+   * the media agents produce in a session: generated images, browser and
+   * computer screenshots, published sandbox files, and generated video
+   * playback. Workspace-level artifact reads must name their session in the
+   * `x-opengeni-session-id` header (`SessionConversation` does this); the proxy
+   * runs `authorizeSession` and only forwards an artifact OpenGeni proves that
+   * session produced. Defaults to true.
+   */
   files?: boolean | undefined;
   /**
    * Forward composer voice input (`POST .../transcriptions`, one recording per
@@ -228,6 +245,18 @@ export type SessionProxyHandlerOptions = {
    * hide the microphone. Defaults to true.
    */
   voiceInput?: boolean | undefined;
+  /**
+   * Forward live speech-to-speech voice for existing chats as the resolved
+   * user: the voice model catalog and the call lifecycle routes under
+   * `.../sessions/{id}/realtime`. Opengeni still requires that user's
+   * `sessions:control` permission, binds the call to that user and browser,
+   * runs spoken requests as ordinary steers of the chat, and meters
+   * deployment-funded voice against the organization's credits. The stock
+   * composer shows the voice button only when a voice model is available.
+   * `false` reports `realtimeVoice: false` in the client config so stock UIs
+   * hide it, and refuses the routes. Defaults to true.
+   */
+  realtimeVoice?: boolean | undefined;
   /**
    * Let the browser read a file from the session's sandbox (`POST .../fs/read`)
    * so `sandbox:` links in agent replies can be downloaded. Only `path`,
@@ -282,10 +311,20 @@ export type SessionProxyHandlerOptions = {
    * settings decide which models end users see.
    */
   modelSelection?: boolean | undefined;
-  /** SSE heartbeat interval. Defaults to 15 seconds. */
+  /**
+   * SSE heartbeat interval. Defaults to 5 seconds, under Bun.serve's default
+   * 10-second `idleTimeout`, which otherwise closes a quiet event stream.
+   */
   heartbeatMs?: number | undefined;
 };
 
+/**
+ * Default SSE heartbeat. Bun.serve closes a connection that sends nothing for
+ * 10 seconds by default, so a quiet stream (a long tool call, an idle chat)
+ * would drop and reconnect every 10 seconds through a Bun or Hono-on-Bun host.
+ */
+const SESSION_PROXY_DEFAULT_HEARTBEAT_MS = 5_000;
+
 const DEFAULT_MAX_BODY_BYTES = 1024 * 1024;
 const SEGMENT = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/;
 const QUEUE_OPERATIONS: ReadonlySet<string> = new Set(["move", "edit", "steer", "delete"]);
@@ -345,6 +384,7 @@ export function createSessionProxyHandler(
   const maxBodyBytes = options.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES;
   const filesEnabled = options.files ?? true;
   const voiceInputEnabled = options.voiceInput ?? true;
+  const realtimeVoiceEnabled = options.realtimeVoice ?? true;
   const sandboxFilesEnabled = options.sandboxFiles === true;
   const artifactsEnabled = options.artifacts !== undefined && options.artifacts !== false;
   const editableLiveUrl = artifactsEnabled
@@ -354,7 +394,7 @@ export function createSessionProxyHandler(
       )
     : null;
   const modelSelection = options.modelSelection ?? true;
-  const heartbeatMs = options.heartbeatMs ?? 15_000;
+  const heartbeatMs = options.heartbeatMs ?? SESSION_PROXY_DEFAULT_HEARTBEAT_MS;
   const sessionList = options.sessionList ?? "mine";
   const archiveEnabled = options.archive ?? true;
   const chats = options.chats ?? "private";
@@ -378,10 +418,10 @@ export function createSessionProxyHandler(
   return async (request) => {
     try {
       const method = request.method;
-      if (!["GET", "POST", "PUT", "PATCH"].includes(method)) {
+      if (!PROXY_METHODS.includes(method)) {
         return new Response(null, {
           status: 405,
-          headers: { Allow: "GET, POST, PUT, PATCH" },
+          headers: { Allow: PROXY_METHODS.join(", ") },
         });
       }
       const resolved = await options.resolve(request);
@@ -511,6 +551,39 @@ export function createSessionProxyHandler(
           ],
         };
       };
+      /**
+       * Best-effort standalone rotation before a voice call. Opengeni refuses
+       * it while a turn is running; that turn's message already refreshed them.
+       */
+      const refreshRealtimeCredentials = async (
+        sessionId: string,
+        updates: SessionMcpCredentialUpdateInput[],
+      ): Promise<void> => {
+        if (updates.length === 0) return;
+        try {
+          const servers = (await client.getSession(workspaceId, sessionId, call)).mcpServers ?? [];
+          const rotations = updates.flatMap((update) => {
+            const server = servers.find((candidate) => candidate.id === update.id);
+            return server
+              ? [
+                  {
+                    id: update.id,
+                    expectedCredentialVersion: server.credentialVersion,
+                    expectedServerUrl: server.url,
+                    headers: update.headers,
+                  },
+                ]
+              : [];
+          });
+          if (rotations.length === 0) return;
+          await client.rotateSessionMcpCredentials(workspaceId, sessionId, {
+            operationKey: crypto.randomUUID(),
+            updates: rotations,
+          });
+        } catch (error) {
+          if (!(error instanceof OpenGeniApiError)) throw error;
+        }
+      };
       /** Browser input sanitized, then server-owned extras merged in. */
       const forwardMessage = async (
         value: unknown,
@@ -602,7 +675,13 @@ export function createSessionProxyHandler(
             // reads it here instead of knowing any Opengeni id.
             workspaceId,
             sandboxFiles: sandboxFilesEnabled,
-            ...(artifacts ? { artifacts } : {}),
+            // Stock UIs show Site previews and the artifact viewer only when
+            // this proxy serves them; `false` keeps them from failing on click.
+            artifacts: artifacts ?? false,
+            // Whether the stock "New chat" and "Archive" actions can succeed.
+            sessionCreation: options.createSession !== undefined,
+            archive: archiveEnabled,
+            ...(realtimeVoiceEnabled ? {} : { realtimeVoice: false }),
             // Explicit true also tells stock UIs to offer end users the model picker.
             ...(modelSelection
               ? options.modelSelection === true
@@ -636,6 +715,10 @@ export function createSessionProxyHandler(
       if (area === "model-catalog" && tail.length === 0 && method === "GET") {
         return await read(`${base}/model-catalog`);
       }
+      if (area === "realtime-model-catalog" && tail.length === 0 && method === "GET") {
+        if (!realtimeVoiceEnabled) return errorJson(404, "route_not_allowed", "Not found.");
+        return await read(`${base}/realtime-model-catalog`);
+      }
       if (area === "live-events" && tail.length === 1 && tail[0] === "stream" && method === "GET") {
         // Surface authorization failures as HTTP status before streaming.
         await client.requestJson("GET", base, undefined, {}, call);
@@ -676,6 +759,53 @@ export function createSessionProxyHandler(
         return errorJson(404, "route_not_allowed", "Not found.");
       }
 
+      if (area === "artifacts") {
+        // Retained media a session produced: only an artifact OpenGeni proves
+        // that session produced, never an arbitrary workspace artifact.
+        const [artifactId, ...artifactOp] = tail as [string | undefined, ...string[]];
+        const route = `${method} ${artifactOp.join("/")}`;
+        if (
+          !filesEnabled ||
+          !artifactId ||
+          !SEGMENT.test(artifactId) ||
+          (route !== "GET content" && route !== "POST playback-source")
+        ) {
+          return errorJson(404, "route_not_allowed", "Not found.");
+        }
+        const sessionId = request.headers.get(SESSION_SCOPE_HEADER)?.trim() ?? "";
+        if (!SEGMENT.test(sessionId)) {
+          reject(400, "session_scope_required", "Artifact reads must name their session.");
+        }
+        if (options.authorizeSession && !(await options.authorizeSession(sessionId, context))) {
+          return errorJson(404, "session_not_found", "Session not found.");
+        }
+        await getSessionProxyArtifactAssociation(
+          client,
+          workspaceId,
+          sessionId,
+          "retained",
+          artifactId,
+          call,
+        );
+        if (route === "POST playback-source") {
+          return json(
+            await client.requestJson(
+              "POST",
+              `${base}/artifacts/${artifactId}/playback-source`,
+              undefined,
+              {},
+              call,
+            ),
+          );
+        }
+        return retainedContent(
+          await client.getRetainedArtifactContent(workspaceId, artifactId, {
+            ...retainedRange(request),
+            signal: request.signal,
+          }),
+        );
+      }
+
       if (area === "transcriptions" && tail.length === 0 && method === "POST") {
         if (!voiceInputEnabled) return errorJson(404, "route_not_allowed", "Not found.");
         return json(await client.transcribeAudio(workspaceId, await transcriptionInput(request)));
@@ -818,6 +948,34 @@ export function createSessionProxyHandler(
           ? payload
           : json(await client.requestJson("POST", path, payload));
 
+      if (op[0] === "realtime") {
+        if (!realtimeVoiceEnabled) return errorJson(404, "route_not_allowed", "Not found.");
+        const realtime = realtimeRoute(method, op.slice(1));
+        if (!realtime) return errorJson(404, "route_not_allowed", "Not found.");
+        if (!body) reject(400, "invalid_body", "A JSON object body is required.");
+        const path = `${session}/${op.join("/")}`;
+        if (realtime === "begin") {
+          // The host may refuse a call; MCP credentials (including the tool
+          // server's per-user token) are refreshed before voice can delegate.
+          const extras = await messageExtras({
+            sessionId,
+            delivery: "realtime",
+          });
+          if (extras instanceof Response) return extras;
+          await refreshRealtimeCredentials(sessionId, extras?.mcpCredentialUpdates ?? []);
+        }
+        let payload: Record<string, unknown> = body;
+        if (realtime === "sync" && options.beforeForwardMessage && hasRealtimeMessages(body)) {
+          const extras = await options.beforeForwardMessage(
+            { sessionId, delivery: "realtime" },
+            context,
+          );
+          if (extras instanceof Response) return extras;
+          payload = withRealtimeContext(body, extras?.modelContext);
+        }
+        return json(await client.requestJson(method, path, payload));
+      }
+
       switch (route) {
         case "GET ":
           return await read(session);
@@ -899,6 +1057,28 @@ export function createSessionProxyHandler(
         }
         case "GET human-input-requests":
           return await read(`${session}/human-input-requests`);
+        case "GET goal":
+          return await read(`${session}/goal`);
+        case "PATCH goal": {
+          // Pause and resume only: the objective, limits and completion stay
+          // with the agent and the product's own server. A browser rationale is
+          // dropped, so end-user text never reaches the goal record.
+          const status = body?.status;
+          if (
+            (status !== "paused" && status !== "active") ||
+            Object.keys(body ?? {}).some((key) => key !== "status" && key !== "rationale")
+          ) {
+            reject(
+              403,
+              "goal_update_not_allowed",
+              "Only { status: paused | active } is available.",
+            );
+          }
+          return json(await client.requestJson("PATCH", `${session}/goal`, { status }));
+        }
+        case "DELETE goal":
+          await client.deleteGoal(workspaceId, sessionId);
+          return new Response(null, { status: 204 });
         case "POST fs/read":
         case "POST fs/read-workspace": {
           if (!sandboxFilesEnabled) return errorJson(404, "route_not_allowed", "Not found.");
@@ -909,6 +1089,21 @@ export function createSessionProxyHandler(
           );
         }
       }
+      if (op[0] === "artifacts" && (op.length === 2 || op[2] === "content") && method === "GET") {
+        // Retained screenshots: the API matches the artifact to this session;
+        // the proxy first proves the user can read the session itself.
+        if (!filesEnabled || op.length > 3) {
+          return errorJson(404, "route_not_allowed", "Not found.");
+        }
+        await client.getSession(workspaceId, sessionId, call);
+        if (op.length === 2) return await read(`${session}/artifacts/${op[1]}`);
+        return retainedContent(
+          await client.getSessionRetainedArtifactContent(workspaceId, sessionId, op[1]!, {
+            ...retainedRange(request),
+            signal: request.signal,
+          }),
+        );
+      }
       if (op.length === 2 && op[0] === "human-input-requests" && method === "GET") {
         return await read(`${session}/human-input-requests/${op[1]}`);
       }
@@ -927,8 +1122,75 @@ export function createSessionProxyHandler(
   };
 }
 
+/**
+ * Methods any forwarded route uses. Module-level so the public API inventory
+ * attributes a verb only to the routes that actually forward it.
+ */
+const PROXY_METHODS: readonly string[] = ["GET", "POST", "PUT", "PATCH", "DELETE"];
+
 const UPLOAD_FIELDS = ["scope", "filename", "contentType", "sizeBytes", "sha256"] as const;
 
+type RealtimeRoute = "begin" | "connect" | "heartbeat" | "end" | "activate" | "sync";
+const REALTIME_CONNECT_ROUTES: ReadonlySet<string> = new Set(["webrtc", "gateway", "supergrok"]);
+/** Ledger entries that become (or join) messages and so accept `modelContext`. */
+const REALTIME_MESSAGE_KINDS: ReadonlySet<string> = new Set([
+  "delegation_call",
+  "user_transcript",
+  "assistant_transcript",
+]);
+
+/**
+ * The live voice routes a browser call uses, after `realtime/`: begin, provider
+ * connect, heartbeat, end, connection activation, and transcript sync. Bodies
+ * pass to Opengeni, which validates them and binds the call to the acting user.
+ */
+function realtimeRoute(method: string, rest: string[]): RealtimeRoute | null {
+  if (rest.length === 0) return method === "POST" ? "begin" : null;
+  if (rest.length === 1) {
+    if (REALTIME_CONNECT_ROUTES.has(rest[0]!)) return method === "POST" ? "connect" : null;
+    return method === "DELETE" ? "end" : null;
+  }
+  if (rest.length === 2 && rest[1] === "heartbeat") return method === "PATCH" ? "heartbeat" : null;
+  if (rest.length === 2 && rest[1] === "sync") return method === "POST" ? "sync" : null;
+  if (rest.length === 4 && rest[1] === "connections" && rest[3] === "activate") {
+    return method === "POST" ? "activate" : null;
+  }
+  return null;
+}
+
+function hasRealtimeMessages(body: Record<string, unknown>): boolean {
+  return (
+    Array.isArray(body.entries) &&
+    body.entries.some(
+      (entry) =>
+        entry !== null &&
+        typeof entry === "object" &&
+        REALTIME_MESSAGE_KINDS.has((entry as { kind?: unknown }).kind as string),
+    )
+  );
+}
+
+/** Server context goes before the browser's on each message-bearing entry. */
+function withRealtimeContext(
+  body: Record<string, unknown>,
+  serverContext: string | undefined,
+): Record<string, unknown> {
+  if (!serverContext?.trim() || !Array.isArray(body.entries)) return body;
+  return {
+    ...body,
+    entries: body.entries.map((entry: unknown) => {
+      if (entry === null || typeof entry !== "object") return entry;
+      const record = entry as Record<string, unknown>;
+      if (!REALTIME_MESSAGE_KINDS.has(record.kind as string)) return entry;
+      const modelContext = joinContext(
+        serverContext,
+        typeof record.modelContext === "string" ? record.modelContext : undefined,
+      );
+      return modelContext ? { ...record, modelContext } : entry;
+    }),
+  };
+}
+
 type NormalizedToolServer = SessionProxyToolServer & {
   url: string;
   id: string;
@@ -1188,6 +1450,35 @@ async function legacyViewerGrant(
   return grant;
 }
 
+/** The browser's byte range, passed through unchanged when well formed. */
+function retainedRange(request: Request): { range?: string } {
+  const range = request.headers.get("range");
+  if (range === null) return {};
+  if (range.length > 128 || /[^\x20-\x7e]/.test(range)) {
+    reject(400, "invalid_range", "Range must be at most 128 printable ASCII bytes.");
+  }
+  return { range };
+}
+
+/** One bounded retained-artifact page, with the range facts the browser SDK verifies. */
+function retainedContent(content: RetainedArtifactContent): Response {
+  return new Response(content.bytes as Uint8Array<ArrayBuffer>, {
+    status: content.status,
+    headers: {
+      "Content-Type": content.contentType,
+      "Content-Length": String(content.contentLength),
+      ...(content.contentRange ? { "Content-Range": content.contentRange } : {}),
+      "Accept-Ranges": "bytes",
+      "Cache-Control": "private, no-store",
+      // Bytes for the conversation's own reader; never rendered on this origin.
+      "Content-Security-Policy": "sandbox",
+      "Content-Disposition": "attachment",
+      "Cross-Origin-Resource-Policy": "same-origin",
+      "X-Content-Type-Options": "nosniff",
+    },
+  });
+}
+
 /** Sandbox link download: one path, no route/target override. */
 function sandboxRead(body: Record<string, unknown> | undefined): Record<string, unknown> {
   const path = body?.path;
@@ -1591,6 +1882,24 @@ function workspaceLiveStream(
   });
 }
 
+let warnedRejectedApiKey = false;
+
+/**
+ * The proxy always calls Opengeni with the server's own key, so an upstream
+ * 401 means that key is wrong, expired or revoked, never that the end user is
+ * signed out. Tell the developer once, in the server log, what to fix.
+ */
+function warnRejectedApiKey(error: OpenGeniApiError): void {
+  if (warnedRejectedApiKey) return;
+  warnedRejectedApiKey = true;
+  console.warn(
+    `[@opengeni/sdk] Opengeni rejected the session proxy's API key (401${
+      error.correlationId ? `, reference ${error.correlationId}` : ""
+    }). It may be expired, revoked or mistyped: create a new key in Opengeni under ` +
+      "Organization settings > Developer and update OPENGENI_API_KEY on the server.",
+  );
+}
+
 /** Preserve OpenGeni's error envelope so the browser SDK keeps codes, retryability, and outcome facts. */
 function errorResponse(error: unknown): Response {
   if (error instanceof ProxyRejection) return errorJson(error.status, error.code, error.message);
@@ -1608,6 +1917,7 @@ function errorResponse(error: unknown): Response {
     );
   }
   if (error instanceof OpenGeniApiError) {
+    if (error.status === 401) warnRejectedApiKey(error);
     const status = error.status >= 400 && error.status <= 599 ? error.status : 502;
     // A decoded upstream envelope is forwarded verbatim; the SDK only retains decodable bodies.
     if (error.body) {

package/src/transcription.ts

@@ -106,6 +106,9 @@ export type TranscriptionLifecycleStatus =
   | "error";
 
 export type TranscriptionErrorCode =
+  | "insufficient_credits"
+  | "allowance_exhausted"
+  | "monthly_model_cost_limit"
   | "permission_denied"
   | "not_supported"
   | "network"