@opengeni/sdk 1.4.0 → 1.4.4

package/dist/index.js

@@ -2,7 +2,7 @@ import {
   EMPTY_FINAL_REPLY_NOTICE,
   OpenGeniEmbeddingClient,
   turnCompletedWithEmptyFinalReply
-} from "./chunk-D534PIXF.js";
+} from "./chunk-MAFL52JD.js";
 import {
   DEFAULT_WORKSPACE_TRANSCRIPTION_POLICY,
   authorizeTranscriptionAdapter,
@@ -35,9 +35,9 @@ import {
   parseCodexRealtimeV3Event,
   projectSessionRealtimeLifecycle,
   startCodexRealtimeWebrtc
-} from "./chunk-ZNC44QEX.js";
-import "./chunk-MXV5O25O.js";
-import "./chunk-CA2T7QT4.js";
+} from "./chunk-JOI7XCG6.js";
+import "./chunk-W72BOLRJ.js";
+import "./chunk-HORQJBLP.js";
 import {
   humanizeModelSlug,
   isRawModelLabel,
@@ -67,7 +67,7 @@ import {
   resumeSequenceFromRequest,
   sessionEventsToSseResponse,
   sessionEventsToSseStream
-} from "./chunk-FYQAMXDW.js";
+} from "./chunk-OLLQBZAE.js";
 import "./chunk-UM4ZXXJR.js";
 import {
   ToolRequestError,
@@ -92,7 +92,7 @@ import {
   isOpenGeniSiteBridgeRequestMessage,
   isSiteCatalogStaleError,
   sanitizeOpenGeniSiteToolCallRequest
-} from "./chunk-LEBDXYMA.js";
+} from "./chunk-7A5ZDWQH.js";
 import {
   OpenGeniToolCallError,
   OpenGeniToolReapprovalRequiredError,
@@ -112,7 +112,7 @@ import {
   streamWorkspaceControlEvents,
   streamWorkspaceInteractionRevisions,
   streamWorkspaceLiveEvents
-} from "./chunk-BVIIRSJY.js";
+} from "./chunk-6SWFOZUP.js";
 import "./chunk-VPVMLM7Q.js";
 import {
   parseSseStream,
@@ -133,7 +133,7 @@ import {
   RETAINED_OUTPUT_MAX_PAGE_BYTES,
   SANDBOX_FILE_ARTIFACT_MAX_BYTES,
   SESSION_EVENT_TYPES
-} from "./chunk-GYM54GKM.js";
+} from "./chunk-DTIMLEPB.js";
 import {
   AttachedBrowserDeviceCollection,
   AttachedBrowserDeviceResource,
@@ -175,7 +175,7 @@ import {
   interactionControlFailureFromError,
   parseBrowserFrameMetadata,
   parseComputerFrameMetadata
-} from "./chunk-NC57YMUO.js";
+} from "./chunk-IJWKWFSF.js";
 import {
   OpenGeniAllowanceExhaustedError,
   OpenGeniApiContractMismatchError,

package/dist/interaction.d.ts

@@ -973,6 +973,8 @@ export type ComputerSessionCapabilities = {
     screenCapture: boolean;
     semanticActions: boolean;
     pointerInput: boolean;
+    /** Causal count-2 input requires its exact confirmed first operation. Absent means unsupported. */
+    pointerClickContinuation?: boolean | undefined;
     keyboardInput: boolean;
     clipboard: boolean;
     backgroundActions: boolean;
@@ -1066,6 +1068,9 @@ export type ComputerAction = {
     deltaX?: number | undefined;
     deltaY?: number | undefined;
     button?: "left" | "right" | "middle" | undefined;
+    /** Click only; 2 delivers one second click pair after a completed first click. */
+    clickCount?: 1 | 2 | undefined;
+    continuationOfOperationId?: string | undefined;
 } | {
     type: "keyboard";
     action: "type" | "press";

package/dist/interaction.js

@@ -39,7 +39,7 @@ import {
   interactionControlFailureFromError,
   parseBrowserFrameMetadata,
   parseComputerFrameMetadata
-} from "./chunk-NC57YMUO.js";
+} from "./chunk-IJWKWFSF.js";
 import "./chunk-V6GIY62B.js";
 export {
   AttachedBrowserDeviceCollection,

package/dist/realtime.js

@@ -23,7 +23,7 @@ import {
   sessionRealtimeOwnerStorageKey,
   sessionRealtimeOwnerStorageNamespace,
   startCodexRealtimeWebrtc
-} from "./chunk-ZNC44QEX.js";
+} from "./chunk-JOI7XCG6.js";
 import {
   createGatewayRealtimeTransportStarter,
   createXaiSubscriptionRealtimeTransportStarter

package/dist/session-proxy.d.ts

@@ -82,14 +82,21 @@ export type SessionProxyCreateInput = {
 export type SessionProxyMessageInput = {
     /** Absent for `create`. */
     sessionId?: string | undefined;
-    delivery: "create" | "send" | "steer" | "submit";
+    /**
+     * `realtime` is live voice: once when a call starts (refuse it, or rotate
+     * MCP credentials), and before each batch of finalized transcripts and spoken
+     * requests is saved (refuse it, or add `modelContext`).
+     */
+    delivery: "create" | "send" | "steer" | "submit" | "realtime";
 };
 /** Server-side additions the host attaches to one forwarded user message or response. */
 export type SessionProxyMessageExtras = {
     /**
      * Model-visible context for this message (current page, time zone, today's
      * date). Placed before any context the browser sent. Not secret. Ignored for
-     * approval decisions and human-input responses, which are not new messages.
+     * approval decisions and human-input responses, which are not new messages,
+     * and when a live voice call starts. Keep it stable for the same message:
+     * a retried voice entry must carry the same context to be accepted.
      */
     modelContext?: string | undefined;
     /**
@@ -157,9 +164,10 @@ export type SessionProxyHandlerOptions = {
     createSession?: ((input: SessionProxyCreateInput, context: SessionProxyContext) => CreateSessionRequest | Response | Promise<CreateSessionRequest | Response>) | undefined;
     /**
      * Called before every forwarded user message (send, steer, composer submit,
-     * and browser-started create), approval decision, and human-input response.
-     * Return server-owned `modelContext` (messages only) and MCP credential
-     * rotations, or a `Response` to reject the action.
+     * and browser-started create), approval decision, human-input response, and
+     * live voice start and transcript save (`delivery: "realtime"`).
+     * Return server-owned `modelContext` (messages and voice transcripts only)
+     * and MCP credential rotations, or a `Response` to reject the action.
      */
     beforeForwardMessage?: ((input: SessionProxyMessageInput, context: SessionProxyContext) => SessionProxyMessageExtras | Response | undefined | Promise<SessionProxyMessageExtras | Response | undefined>) | undefined;
     /**
@@ -192,6 +200,18 @@ export type SessionProxyHandlerOptions = {
      * hide the microphone. Defaults to true.
      */
     voiceInput?: boolean | undefined;
+    /**
+     * Forward live speech-to-speech voice for existing chats as the resolved
+     * user: the voice model catalog and the call lifecycle routes under
+     * `.../sessions/{id}/realtime`. Opengeni still requires that user's
+     * `sessions:control` permission, binds the call to that user and browser,
+     * runs spoken requests as ordinary steers of the chat, and meters
+     * deployment-funded voice against the organization's credits. The stock
+     * composer shows the voice button only when a voice model is available.
+     * `false` reports `realtimeVoice: false` in the client config so stock UIs
+     * hide it, and refuses the routes. Defaults to true.
+     */
+    realtimeVoice?: boolean | undefined;
     /**
      * Let the browser read a file from the session's sandbox (`POST .../fs/read`)
      * so `sandbox:` links in agent replies can be downloaded. Only `path`,
@@ -248,7 +268,10 @@ export type SessionProxyHandlerOptions = {
      * settings decide which models end users see.
      */
     modelSelection?: boolean | undefined;
-    /** SSE heartbeat interval. Defaults to 15 seconds. */
+    /**
+     * SSE heartbeat interval. Defaults to 5 seconds, under Bun.serve's default
+     * 10-second `idleTimeout`, which otherwise closes a quiet event stream.
+     */
     heartbeatMs?: number | undefined;
 };
 /**

package/dist/session-proxy.js

@@ -6,11 +6,11 @@ import {
   downloadSessionProxySiteHtml,
   getSessionProxyArtifactAssociation,
   getSessionProxyWorkspaceGrant
-} from "./chunk-FYQAMXDW.js";
+} from "./chunk-OLLQBZAE.js";
 import "./chunk-UM4ZXXJR.js";
 import "./chunk-U6CV2SXO.js";
 import "./chunk-APXVSSSY.js";
-import "./chunk-GYM54GKM.js";
+import "./chunk-DTIMLEPB.js";
 import "./chunk-V6GIY62B.js";
 export {
   SESSION_PROXY_SITE_HTML_MAX_BYTES,

package/dist/site.js

@@ -18,13 +18,13 @@ import {
   sanitizeOpenGeniSiteToolCallRequest,
   serveSiteHttp,
   siteSessionPath
-} from "./chunk-LEBDXYMA.js";
+} from "./chunk-7A5ZDWQH.js";
 import "./chunk-4SMEOV3P.js";
-import "./chunk-BVIIRSJY.js";
+import "./chunk-6SWFOZUP.js";
 import "./chunk-VPVMLM7Q.js";
 import "./chunk-APXVSSSY.js";
-import "./chunk-GYM54GKM.js";
-import "./chunk-NC57YMUO.js";
+import "./chunk-DTIMLEPB.js";
+import "./chunk-IJWKWFSF.js";
 import "./chunk-V6GIY62B.js";
 export {
   OPENGENI_SITE_BRIDGE_BOOTSTRAP_GLOBAL,

package/dist/types.d.ts

@@ -3341,6 +3341,13 @@ export type ClientConfig = {
     modelSelection?: boolean | undefined;
     /** Session proxy sandbox-path download opt-in; absent on native deployments. */
     sandboxFiles?: boolean | undefined;
+    /**
+     * `false` when a host's session proxy turns live voice off
+     * (`createSessionProxyHandler({ realtimeVoice: false })`), so UIs hide the
+     * voice button. Otherwise absent: availability comes from the workspace's
+     * realtime model catalog.
+     */
+    realtimeVoice?: boolean | undefined;
     /**
      * Session proxy only: the workspace the proxy resolved for this user, so a
      * browser pointed at the proxy (`<OpenGeniChat baseUrl=... />`) needs no
@@ -5948,6 +5955,14 @@ export type CapabilityRuntime = {
         state: "trusted" | "legacy_active" | "unverified";
         reason: "trusted_source" | "verified_probe" | "active_installation_compatibility" | "missing_verification";
     } | undefined;
+    /**
+     * Present when connecting needs an operator-registered OAuth client because
+     * the provider refuses self-registration; `configured` is whether this
+     * deployment has one. Connector surfaces offer the row only when true.
+     */
+    operatorOAuthClient?: {
+        configured: boolean;
+    } | undefined;
 };
 export type CapabilityCatalogItem = {
     id: string;
@@ -6695,6 +6710,8 @@ export type PromotionalCreditScope = {
 export type PromotionalCreditBalance = PromotionalCreditScope & {
     grantId: string;
     remainingMicros: number;
+    /** Also pays for dictation and live voice, like general credits (signup credits). */
+    coversVoice?: boolean | undefined;
 };
 export type BillingBalance = {
     accountId: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opengeni/sdk",
-  "version": "1.4.0",
+  "version": "1.4.4",
   "description": "Framework-agnostic TypeScript SDK for the OpenGeni API: typed client, session lifecycle, SSE event streaming with reconnect + replay-by-sequence, and proxy re-streaming helpers.",
   "license": "Apache-2.0",
   "repository": {
@@ -182,8 +182,8 @@
     "prepublishOnly": "bash ../../scripts/prepublish-guard"
   },
   "dependencies": {
-    "@opengeni/connect": "1.4.0",
-    "@opengeni/contracts": "1.4.0"
+    "@opengeni/connect": "1.4.4",
+    "@opengeni/contracts": "1.4.4"
   },
   "engines": {
     "node": ">=18"

package/src/chat/opengeni.ts

@@ -31,6 +31,13 @@ export const DEFAULT_CHAT_SOURCE = "app";
 
 type SubmittedTurn = { after: number; turnId: string | null };
 
+const MISSING_API_KEY_MESSAGE =
+  "Opengeni requires an apiKey. Set OPENGENI_API_KEY in the server environment.";
+
+async function missingApiKeyFetch(): Promise<Response> {
+  throw new TypeError(MISSING_API_KEY_MESSAGE);
+}
+
 type BuildCreate = (text: string, send: ChatSendOptions) => CreateSessionRequest;
 type SubmitCreate = (request: CreateSessionRequest) => Promise<CreateSessionResponse>;
 
@@ -74,12 +81,16 @@ export class OpenGeni {
   private pendingOrganizationId: Promise<string> | undefined;
 
   constructor(options: OpenGeniOptions) {
-    if (!options.apiKey) throw new TypeError("Opengeni requires an apiKey.");
     this.client = new OpenGeniClient({
       // An empty value (a blank `OPENGENI_API_BASE_URL=` in .env) means the default.
       baseUrl: options.baseUrl?.trim() || DEFAULT_OPENGENI_BASE_URL,
-      apiKey: options.apiKey,
-      ...(options.fetch ? { fetch: options.fetch } : {}),
+      // A missing key fails each request, not construction: a module-scope
+      // `new OpenGeni({ apiKey: process.env.OPENGENI_API_KEY! })` must not
+      // break `next build` (or any import) where the secret only exists at
+      // runtime.
+      ...(options.apiKey
+        ? { apiKey: options.apiKey, ...(options.fetch ? { fetch: options.fetch } : {}) }
+        : { fetch: missingApiKeyFetch }),
     });
     this.organizationId = options.organizationId?.trim() ?? "";
     this.source = options.source ?? DEFAULT_CHAT_SOURCE;

package/src/codex-realtime-controller.ts

@@ -142,15 +142,7 @@ const REALTIME_REFUSAL_CODES = new Set([
 export function codexRealtimeRefusal(error: unknown): CodexRealtimeRefusal | null {
   if (!(error instanceof OpenGeniApiError) || !error.code) return null;
   if (!REALTIME_REFUSAL_CODES.has(error.code)) return null;
-  let message: string | null = null;
-  try {
-    const body = JSON.parse(error.body) as Record<string, unknown>;
-    const nested =
-      body.error && typeof body.error === "object" ? (body.error as Record<string, unknown>) : body;
-    message = typeof nested.message === "string" && nested.message ? nested.message : null;
-  } catch {
-    message = null;
-  }
+  const message = apiErrorMessage(error);
   return {
     code: error.code,
     message:
@@ -412,6 +404,9 @@ export function createCodexRealtimeController(
   let generation = 0;
   let reconnectAttempt = 0;
   let recoveryTerminal = false;
+  // Whether this mode ever reached a live provider connection. A definitive
+  // failure before that ends the mode instead of leaving an empty call open.
+  let connectedInMode = false;
   let mutationTail = Promise.resolve();
   let connectionTask: Promise<void> | null = null;
   const acceptedDelegationItemIds = new Set(
@@ -740,6 +735,12 @@ export function createCodexRealtimeController(
     if (closed || stopping || isAbortError(error)) return;
     const message = safeError(error);
     if (error instanceof CodexRealtimeMicrophoneError) {
+      if (error.code !== "track_ended" && !connectedInMode) {
+        // No conversation exists yet: end the call and say how to fix the
+        // microphone instead of holding an empty "reconnecting" call open.
+        await endAfterFailure(microphoneFailureMessage(error), null);
+        return;
+      }
       if (error.code === "track_ended" && state.mode?.state === "active") {
         reconnectAttempt += 1;
         publish({
@@ -783,6 +784,9 @@ export function createCodexRealtimeController(
           return;
         }
       }
+    } else if (error instanceof OpenGeniApiError && !error.retryable && !connectedInMode) {
+      await endAfterFailure(message, null);
+      return;
     } else if (error instanceof OpenGeniApiError && !error.retryable) {
       stopTimers();
       if (!active) releaseMicrophone();
@@ -1071,6 +1075,7 @@ export function createCodexRealtimeController(
         onFatal: (fatal) => onBridgeFatal(targetGeneration, bridge, fatal),
       });
       active = { generation: targetGeneration, transport: connected, bridge };
+      connectedInMode = true;
       connected.setOutputMuted(state.outputMuted);
       recoveryTerminal = false;
       pendingAbort = null;
@@ -1190,7 +1195,13 @@ export function createCodexRealtimeController(
    * End the call because Opengeni refused it, keep final speech, and leave a
    * terminal, non-retrying state that explains why.
    */
-  const endForRefusal = async (refusal: CodexRealtimeRefusal): Promise<void> => {
+  const endForRefusal = async (refusal: CodexRealtimeRefusal): Promise<void> =>
+    await endAfterFailure(refusal.message, refusal);
+
+  const endAfterFailure = async (
+    message: string,
+    refusal: CodexRealtimeRefusal | null,
+  ): Promise<void> => {
     try {
       await controller.stop();
     } catch {
@@ -1205,8 +1216,8 @@ export function createCodexRealtimeController(
       realtimeId: null,
       mode: null,
       bridge: null,
-      diagnostic: diagnostic("terminal_stop", refusal.message, false),
-      error: refusal.message,
+      diagnostic: diagnostic("terminal_stop", message, false),
+      error: message,
       refusal,
     });
   };
@@ -1263,6 +1274,7 @@ export function createCodexRealtimeController(
       closed = false;
       stopping = false;
       recoveryTerminal = false;
+      connectedInMode = false;
       const record: OwnerRecord = {
         version: OWNER_RECORD_VERSION,
         workspaceId: options.workspaceId,
@@ -1303,6 +1315,8 @@ export function createCodexRealtimeController(
         if (recoveryTerminal && state.mode?.state === "active") return;
         const record = readOwnerRecord(storage, storageKey, options);
         if (!record) {
+          // Keep a terminal failure readable after the server records the end.
+          if (state.status === "error" && !state.mode) return;
           transitionEnded();
           return;
         }
@@ -1326,7 +1340,10 @@ export function createCodexRealtimeController(
       if (lifecycle.state === "ended") {
         const record = readOwnerRecord(storage, storageKey, options);
         if (record && record.operationId !== lifecycle.operationId) {
-          if (connectionTask || state.status === "active") return;
+          // An earlier call's end is not news while this browser is starting
+          // its own call (for example, waiting on the microphone prompt):
+          // re-beginning would advance the lease and fail the pending start.
+          if (connectionTask || state.status === "active" || state.status === "starting") return;
           closed = false;
           stopping = false;
           owner = record;
@@ -1342,6 +1359,9 @@ export function createCodexRealtimeController(
           await connectionTask;
           return;
         }
+        // The server's end of a call that failed to start must not wipe the
+        // failure the user still needs to read.
+        if (state.status === "error" && !state.mode) return;
         if (state.realtimeId === null || lifecycle.realtimeId === state.realtimeId) {
           transitionEnded(`Realtime ended: ${lifecycle.reason}`);
         }
@@ -1801,6 +1821,37 @@ function isAbortError(error: unknown): boolean {
   return error instanceof Error && error.name === "AbortError";
 }
 
+function microphoneFailureMessage(error: CodexRealtimeMicrophoneError): string {
+  switch (error.code) {
+    case "permission_denied":
+      return "Microphone access is blocked. Allow it in site settings, then try again.";
+    case "device_not_found":
+      return "No microphone was found. Connect one, then try again.";
+    case "device_unavailable":
+      return "Your microphone is busy or unavailable. Close other apps using it, then try again.";
+    default:
+      return /timed out/i.test(error.message)
+        ? "Microphone access wasn't granted in time. Allow it when your browser asks, then try again."
+        : "The microphone could not start. Try again.";
+  }
+}
+
+/** The server's own message, without the transport prefix or reference id. */
+function apiErrorMessage(error: OpenGeniApiError): string | null {
+  try {
+    const body = JSON.parse(error.body) as Record<string, unknown>;
+    const nested =
+      body.error && typeof body.error === "object" ? (body.error as Record<string, unknown>) : body;
+    return typeof nested.message === "string" && nested.message ? nested.message : null;
+  } catch {
+    return null;
+  }
+}
+
 function safeError(error: unknown): string {
-  return error instanceof Error ? error.message : "Codex realtime browser controller failed";
+  if (error instanceof OpenGeniApiError) {
+    const message = apiErrorMessage(error);
+    if (message) return /[.!?]$/.test(message) ? message : `${message}.`;
+  }
+  return error instanceof Error ? error.message : "Live voice failed in this browser.";
 }

package/src/interaction.ts

@@ -1114,6 +1114,8 @@ export type ComputerSessionCapabilities = {
   screenCapture: boolean;
   semanticActions: boolean;
   pointerInput: boolean;
+  /** Causal count-2 input requires its exact confirmed first operation. Absent means unsupported. */
+  pointerClickContinuation?: boolean | undefined;
   keyboardInput: boolean;
   clipboard: boolean;
   backgroundActions: boolean;
@@ -1217,6 +1219,9 @@ export type ComputerAction =
       deltaX?: number | undefined;
       deltaY?: number | undefined;
       button?: "left" | "right" | "middle" | undefined;
+      /** Click only; 2 delivers one second click pair after a completed first click. */
+      clickCount?: 1 | 2 | undefined;
+      continuationOfOperationId?: string | undefined;
     }
   | { type: "keyboard"; action: "type" | "press"; value: string }
   | {