@pyai/sdk 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -31,13 +31,13 @@ import PyAI from "@pyai/sdk";
31
31
  const pyai = new PyAI({ apiKey: process.env.PYAI_API_KEY! });
32
32
 
33
33
  // Text-to-speech
34
- const audio = await pyai.audio.speech({ input: "Hello from PyAI.", voice: "stock_sarah_style2" });
34
+ const audio = await pyai.audio.speech({ input: "Hello from PyAI.", voice: "stock_emma_en_gb" });
35
35
  await Bun.write?.("hello.wav", audio); // or fs.writeFile in Node
36
36
 
37
37
  // Text-to-speech, streamed — start playing/forwarding at the first chunk
38
38
  // (tens of ms) instead of waiting for the whole clip. Use mp3 for smooth
39
39
  // progressive playback.
40
- const stream = await pyai.audio.speechStream({ input: "Hello from PyAI.", voice: "stock_sarah_style2", response_format: "mp3" });
40
+ const stream = await pyai.audio.speechStream({ input: "Hello from PyAI.", voice: "stock_emma_en_gb", response_format: "mp3" });
41
41
  for await (const chunk of stream) writeToSpeakerOrResponse(chunk);
42
42
 
43
43
  // Voices
@@ -53,20 +53,42 @@ const done = await pyai.transcriptionJobs.get(job.job_id);
53
53
 
54
54
  ## Realtime (Omni)
55
55
 
56
- Keys travel as a WebSocket subprotocol so this works in the browser:
56
+ `omni.connect()` opens an agentic-voice session and hides the wire protocol
57
+ including its **frame-key asymmetry** (your control frames are keyed on `type`,
58
+ the server's frames are keyed on `event`). It sends a `type`-keyed `configure`
59
+ the instant the socket opens and routes server frames to typed callbacks, so you
60
+ **can't** trip the #1 Omni integration bug (a hand-rolled `{"event":"configure"}`
61
+ is acked but silently dropped, giving you a connected session with zero turns):
57
62
 
58
63
  ```ts
59
- const ws = pyai.connectRealtime({ product: "omni", agentId: "agent_123" });
60
- ws.addEventListener("message", (e) => console.log(e.data));
64
+ // Omni is zero-state: the key's org authorizes the session — nothing to create.
65
+ const omni = pyai.omni.connect({
66
+ rate: 16000, // 24000 browser · 16000 wideband telephony · 8000 G.711/Twilio
67
+ configure: { voice_id: "stock_emma_en_gb", persona: "You are a receptionist." },
68
+ onAudio: (chunk) => speaker.write(chunk), // binary agent audio — play it out
69
+ onTranscript: (f) => console.log(f.text),
70
+ onError: (e) => console.error(e),
71
+ });
72
+
73
+ omni.sendAudio(pcm16Chunk); // stream caller audio continuously (server-side VAD)
74
+ omni.sendDtmf("5");
75
+ omni.close();
76
+ ```
77
+
78
+ From the browser, mint an ephemeral token server-side with
79
+ `pyai.omni.createSession({ allowedOrigins })` and pass it as `token` so the page
80
+ never holds a secret key:
61
81
 
62
- // Or build the pieces yourself for a custom WS library:
63
- const url = pyai.realtimeURL({ product: "omni", agentId: "agent_123" });
64
- const proto = pyai.realtimeSubprotocol();
82
+ ```ts
83
+ const omni = pyai.omni.connect({ token: session.token, configure: { voice_id, persona } });
65
84
  ```
66
85
 
67
- > Omni uses the native `wss://api.pyai.com/v1/omni` surface. `connectRealtime`
68
- > targets it by default (`product: "omni"`); `product: "flow"` uses
69
- > `/v1/realtime`. The older `/v2/omni/chat` URL is deprecated but still works.
86
+ > Omni uses the native `wss://api.pyai.com/v1/omni` surface and is **zero-state**
87
+ > no agent to create, `sessionLabel` is an optional opaque tag (never
88
+ > required). Need the raw socket? `pyai.realtimeURL({ product: "omni" })` +
89
+ > `pyai.realtimeSubprotocol()` (or `pyai.connectRealtime()`) still work;
90
+ > `product: "flow"` uses `/v1/realtime`. The older `/v2/omni/chat` URL and the
91
+ > `agentId` option are deprecated but still work.
70
92
 
71
93
  ## Streaming speech-to-text (Hear / Cue)
72
94
 
@@ -116,7 +138,7 @@ drop the hand-rolled resampler + μ-law encoder entirely:
116
138
  // Twilio/SIP-ready in one param: raw 8 kHz mono μ-law, no client-side DSP.
117
139
  const ulaw = await pyai.audio.speech({
118
140
  input: "Your appointment is confirmed.",
119
- voice: "stock_sarah_style2",
141
+ voice: "stock_emma_en_gb",
120
142
  response_format: "g711_ulaw", // -> audio/basic, forced 8 kHz
121
143
  });
122
144
  // base64-encode `ulaw` straight into a Twilio media frame.
@@ -176,7 +198,7 @@ const quality = detail.quality_metrics; // { wer?, ttfb_ms?,
176
198
  once the engine supports them and otherwise ignored — so it's always safe to send:
177
199
 
178
200
  ```ts
179
- await pyai.audio.speech({ input: "Hello", voice: "stock_sarah_style2", seed: 42, temperature: 0 });
201
+ await pyai.audio.speech({ input: "Hello", voice: "stock_emma_en_gb", seed: 42, temperature: 0 });
180
202
  await pyai.audio.transcriptions.create({ file: wavBlob, seed: 42 });
181
203
  ```
182
204
 
@@ -225,7 +247,7 @@ Other commands:
225
247
  ```bash
226
248
  pyai models
227
249
  pyai voices --gender female --region en_us
228
- pyai speak --text "Hello" --voice stock_sarah_style2 --out hello.wav
250
+ pyai speak --text "Hello" --voice stock_emma_en_gb --out hello.wav
229
251
  pyai transcribe --url https://example.com/call.wav --diarize --poll
230
252
  ```
231
253
 
package/dist/index.d.ts CHANGED
@@ -121,13 +121,52 @@ export interface CreateJobParams {
121
121
  webhook_url?: string;
122
122
  }
123
123
  export interface RealtimeOptions {
124
- /** "omni" (agentic, needs agentId) or "flow" (voice duplex). Default "omni". */
124
+ /** "omni" (agentic voice) or "flow" (legacy voice duplex). Default "omni". */
125
125
  product?: "omni" | "flow";
126
- /** Required for omni: the agent to drive. */
126
+ /**
127
+ * Optional opaque tag echoed to your `kb_endpoint` and recorded on the call.
128
+ * Omni is zero-state: the session is authorized by the key's org, so there is
129
+ * nothing to create first and this tag is never required.
130
+ */
131
+ sessionLabel?: string;
132
+ /**
133
+ * @deprecated Use {@link sessionLabel}. Kept for back-compat — emitted as the
134
+ * `agent_id` query alias, which the gateway still accepts. Ignored if
135
+ * `sessionLabel` is set.
136
+ */
127
137
  agentId?: string;
128
138
  /** Extra query params (e.g. format, rate). */
129
139
  query?: Record<string, string>;
130
140
  }
141
+ /** Parameters for minting an ephemeral browser Omni session token (server-side). */
142
+ export interface OmniSessionParams {
143
+ /**
144
+ * Browser origins (scheme://host[:port]) the minted token may connect from.
145
+ * Required and non-empty — a browser token must be origin-locked. `*` is not
146
+ * allowed.
147
+ */
148
+ allowedOrigins: string[];
149
+ /** Token lifetime in seconds. Default 60 server-side; max 600. Keep it short. */
150
+ ttlSeconds?: number;
151
+ /** Optional opaque tag echoed back to your `kb_endpoint` and recorded on the call. */
152
+ sessionLabel?: string;
153
+ }
154
+ /**
155
+ * The result of {@link PyAI.omni}.createSession — a short-lived credential safe
156
+ * to hand to the browser. Use {@link OmniSession.token} as the WebSocket
157
+ * subprotocol `pyai-key.<token>` against {@link OmniSession.url}.
158
+ */
159
+ export interface OmniSession {
160
+ object?: "omni.session";
161
+ /** The ephemeral session token. Short-lived + origin-locked; browser-safe. */
162
+ token: string;
163
+ /** Token expiry, Unix epoch milliseconds. */
164
+ expires_at: number;
165
+ /** The Omni realtime WebSocket URL to connect to. */
166
+ url: string;
167
+ /** Echoed back when supplied on the request. */
168
+ session_label?: string;
169
+ }
131
170
  /** Frame `type`s emitted by the Hear streaming-STT WebSocket. */
132
171
  export declare const HearFrameType: {
133
172
  /** Eager live hypothesis for the current utterance. */
@@ -138,6 +177,8 @@ export declare const HearFrameType: {
138
177
  readonly SpeechFinal: "speech_final";
139
178
  /** Corrected, full-context transcript following `speech_final`. */
140
179
  readonly Final: "final";
180
+ /** Final billed-usage summary, emitted just before a graceful close. */
181
+ readonly Usage: "usage";
141
182
  /** Server-side fault frame. */
142
183
  readonly Error: "error";
143
184
  };
@@ -203,13 +244,28 @@ export interface HearFinalFrame {
203
244
  /** Present only with Cue grounding enabled (top KB passages). */
204
245
  grounding?: HearGroundingPassage[];
205
246
  }
247
+ /** Final billed-usage summary (`usage`), emitted just before a graceful close
248
+ * so realtime spend can be reconciled in-band (a realtime WS carries no
249
+ * `x-pyai-units` response header). Best-effort: absent if there was no billable
250
+ * audio or the close was abnormal. */
251
+ export interface HearUsageFrame {
252
+ type: "usage";
253
+ /** `hear` for plain streaming, `cue` when grounding was enabled. */
254
+ product: "hear" | "cue";
255
+ /** The billed meter (`hear.requests` or `cue.minutes`). */
256
+ meter: string;
257
+ /** Summed active-speech audio billed for the session, in seconds. */
258
+ audio_seconds: number;
259
+ /** The same quantity in minutes. */
260
+ minutes: number;
261
+ }
206
262
  /** Server fault frame (`error`). */
207
263
  export interface HearErrorFrame {
208
264
  type: "error";
209
265
  code?: string;
210
266
  message: string;
211
267
  }
212
- export type HearFrame = HearPartialFrame | HearFinalFrame | HearErrorFrame;
268
+ export type HearFrame = HearPartialFrame | HearFinalFrame | HearUsageFrame | HearErrorFrame;
213
269
  /**
214
270
  * Minimal structural WebSocket — matches both the browser/Node global
215
271
  * `WebSocket` and the `ws` package, and lets tests inject a mock.
@@ -240,12 +296,32 @@ export interface HearStreamOptions {
240
296
  encoding?: "pcm16" | "opus";
241
297
  /** Emit eager partial hypotheses. Default true server-side. */
242
298
  interimResults?: boolean;
299
+ /**
300
+ * Format spoken numbers as digits in the transcript (e.g. "one two three" →
301
+ * "123"). Useful for voice agents that read back phone numbers, codes, and
302
+ * amounts. Default false (spoken form). Forwards `?numerals=true` on the URL.
303
+ */
304
+ numerals?: boolean;
305
+ /**
306
+ * Turn-segmentation tuning: trailing-pause (ms, 50–2000) that ends an
307
+ * utterance. Forwards `?endpointing_ms=` — clamped + honored once the engine
308
+ * supports it; a no-op when omitted. Drive end-of-turn yourself with
309
+ * {@link HearStream.commit} for full control today.
310
+ */
311
+ endpointingMs?: number;
243
312
  /**
244
313
  * Enable Cue knowledge-base grounding: sends `{type:"config",grounding:true}`
245
314
  * on open, after which `speech_final`/`final` frames carry a `grounding`
246
315
  * array. Bills a single `cue.minutes` line instead of the Hear rate.
247
316
  */
248
317
  grounding?: boolean;
318
+ /** Cue: number of KB passages to retrieve per turn (1–20, default 3). */
319
+ groundingK?: number;
320
+ /** Cue: drop passages scoring below this (0–1, default 0 = keep all). */
321
+ groundingMinScore?: number;
322
+ /** Cue: max ms to wait for retrieval at the final before failing open to
323
+ * `grounding: []` (50–2000, default 450). */
324
+ groundingTimeoutMs?: number;
249
325
  /** Extra query params merged onto the connect URL. */
250
326
  query?: Record<string, string>;
251
327
  /** Fired once the socket opens (after the optional grounding config). */
@@ -254,6 +330,8 @@ export interface HearStreamOptions {
254
330
  onPartial?: (frame: HearPartialFrame) => void;
255
331
  /** Fired on `speech_final` / `final`. */
256
332
  onFinal?: (frame: HearFinalFrame) => void;
333
+ /** Fired on the final `usage` summary (in-band realtime reconciliation). */
334
+ onUsage?: (frame: HearUsageFrame) => void;
257
335
  /** Fired on an `error` frame or a transport-level error. */
258
336
  onError?: (err: HearErrorFrame | Error) => void;
259
337
  /** Fired when the socket closes (code per `WSCloseCode`). */
@@ -284,6 +362,152 @@ export declare class HearStream {
284
362
  /** Current WebSocket readyState. */
285
363
  get readyState(): number;
286
364
  }
365
+ /**
366
+ * Event names on Omni **server → client** frames. ⚠️ Note the asymmetry:
367
+ * inbound frames are keyed on `event`, but your **outbound** control frames
368
+ * (`configure`, `dtmf`, …) are keyed on `type`. {@link OmniConnection} handles
369
+ * both sides for you; this map is for matching frames in `onEvent`.
370
+ */
371
+ export declare const OmniEvent: {
372
+ /** Handshake; advertises protocol version + audio formats. */
373
+ readonly Hello: "hello";
374
+ /** Ack for your `configure` frame (echoes the resolved `voice_id`). */
375
+ readonly Configured: "configured";
376
+ /** Session is live; includes the resolved agent + audio caps. */
377
+ readonly SessionStarted: "session_started";
378
+ /** Turn boundary (user/assistant speaking). */
379
+ readonly Turn: "turn";
380
+ /** Incremental/final transcript text. */
381
+ readonly Transcript: "transcript";
382
+ /** User interrupted; assistant audio is being cut. */
383
+ readonly BargeIn: "barge_in";
384
+ /** Alias for barge-in on some engine builds. */
385
+ readonly Flush: "flush";
386
+ /** Session is closing; see close code. */
387
+ readonly SessionEnd: "session_end";
388
+ /** Server fault frame. */
389
+ readonly Error: "error";
390
+ };
391
+ export type OmniEvent = (typeof OmniEvent)[keyof typeof OmniEvent];
392
+ /** A server → client Omni frame. Keyed on `event` (not `type`); open for forward-compat. */
393
+ export interface OmniServerFrame {
394
+ event: string;
395
+ [k: string]: unknown;
396
+ }
397
+ /** A binary agent-audio chunk delivered to {@link OmniConnectOptions.onAudio}. */
398
+ export type OmniAudioChunk = ArrayBuffer | ArrayBufferView | Blob;
399
+ /**
400
+ * The agent config sent as the `configure` control frame. These are the wire
401
+ * (snake_case) fields the engine reads; it's an **open bag**, so forward-compat
402
+ * fields (e.g. `greeting` / `language` once the engine acks them) pass straight
403
+ * through. The SDK supplies the `{"type":"configure"}` envelope for you — which
404
+ * is the whole point: a hand-rolled `{"event":"configure"}` is acked but
405
+ * silently dropped, leaving the agent with no brain and zero turns.
406
+ */
407
+ export interface OmniConfigure {
408
+ /** Voice to speak with (stock / clone / designed id). */
409
+ voice_id?: string;
410
+ /** System prompt / role + instructions for the agent. */
411
+ persona?: string;
412
+ /** Customer-hosted URL the engine calls per turn for grounding. */
413
+ kb_endpoint?: string;
414
+ /** Bearer the engine presents to `kb_endpoint`. */
415
+ kb_token?: string;
416
+ /** Forward-compatible: any other key the engine honors. */
417
+ [k: string]: unknown;
418
+ }
419
+ export interface OmniConnectOptions {
420
+ /** Optional opaque per-session tag echoed to your `kb_endpoint`. Omni is zero-state. */
421
+ sessionLabel?: string;
422
+ /**
423
+ * Ephemeral browser token from `omni.createSession`. When set it is used as
424
+ * the WS subprotocol (`pyai-key.<token>`) instead of the client's secret key,
425
+ * so a page never holds a secret key.
426
+ */
427
+ token?: string;
428
+ /** Connect-URL sample rate: 24000 browser, 16000 wideband telephony, 8000 G.711/Twilio. */
429
+ rate?: 24000 | 16000 | 8000;
430
+ /** Connect-URL audio format. Default `pcm16`. */
431
+ format?: "pcm16";
432
+ /**
433
+ * Agent config sent as `{"type":"configure",...}` the instant the socket
434
+ * opens. Omit to send it yourself later via {@link OmniConnection.configure}.
435
+ */
436
+ configure?: OmniConfigure;
437
+ /** Extra query params merged onto the connect URL. */
438
+ query?: Record<string, string>;
439
+ /** Fired once the socket opens (after the optional auto-configure). */
440
+ onOpen?: () => void;
441
+ /** Fired for each binary agent-audio chunk — play it out as it arrives. */
442
+ onAudio?: (chunk: OmniAudioChunk) => void;
443
+ /** Fired on the `hello` handshake frame. */
444
+ onHello?: (frame: OmniServerFrame) => void;
445
+ /** Fired on the `configured` ack. */
446
+ onConfigured?: (frame: OmniServerFrame) => void;
447
+ /** Fired on `session_started`. */
448
+ onSessionStarted?: (frame: OmniServerFrame) => void;
449
+ /** Fired on `turn` boundaries. */
450
+ onTurn?: (frame: OmniServerFrame) => void;
451
+ /** Fired on `transcript` text frames. */
452
+ onTranscript?: (frame: OmniServerFrame) => void;
453
+ /** Fired on `barge_in` / `flush` (user interrupted). */
454
+ onBargeIn?: (frame: OmniServerFrame) => void;
455
+ /** Fired on `session_end`. */
456
+ onSessionEnd?: (frame: OmniServerFrame) => void;
457
+ /** Fired on EVERY JSON frame (including unknown/forward-compat ones). */
458
+ onEvent?: (frame: OmniServerFrame) => void;
459
+ /** Fired on an `error` frame or a transport-level error. */
460
+ onError?: (err: OmniServerFrame | Error) => void;
461
+ /** Fired when the socket closes (code per {@link WSCloseCode}). */
462
+ onClose?: (code: number, reason: string) => void;
463
+ /** Injectable WebSocket constructor (defaults to the global). */
464
+ webSocket?: WebSocketCtor;
465
+ }
466
+ /**
467
+ * A live Omni agentic-voice session over `/v1/omni`. Hides the wire protocol's
468
+ * frame-key asymmetry: it sends control frames keyed on `type` (`configure`,
469
+ * `dtmf`) and parses server frames keyed on `event`, so you cannot trip the #1
470
+ * Omni integration bug (mirroring the server's `event` key on outbound, which
471
+ * is silently dropped). Construct via `pyai.omni.connect()`.
472
+ *
473
+ * @example
474
+ * const omni = pyai.omni.connect({
475
+ * rate: 16000,
476
+ * configure: { voice_id: "stock_emma_en_gb", persona: "You are a receptionist." },
477
+ * onAudio: (chunk) => speaker.write(chunk),
478
+ * onTranscript: (f) => console.log(f.text),
479
+ * });
480
+ * omni.sendAudio(pcm16Chunk); // stream caller audio continuously
481
+ */
482
+ export declare class OmniConnection {
483
+ private readonly ws;
484
+ private readonly opts;
485
+ private closed;
486
+ constructor(url: string, subprotocol: string, opts: OmniConnectOptions);
487
+ private handleMessage;
488
+ /**
489
+ * Send (or update) the agent config. Always emitted as
490
+ * `{"type":"configure", ...}` — the correct key. (A hand-rolled
491
+ * `{"event":"configure"}` is acked but silently dropped by the engine.)
492
+ */
493
+ configure(cfg: OmniConfigure): void;
494
+ /** Stream a chunk of caller audio (PCM16 LE mono at the negotiated rate). */
495
+ sendAudio(chunk: ArrayBufferLike | ArrayBufferView | Blob): void;
496
+ /** Send DTMF digits as a `{"type":"dtmf"}` control frame. */
497
+ sendDtmf(digits: string): void;
498
+ /**
499
+ * Send an arbitrary control frame for forward-compat control types the SDK
500
+ * does not model yet. Reminder: client → server frames are keyed on `type`,
501
+ * never `event`.
502
+ */
503
+ send(frame: Record<string, unknown>): void;
504
+ /** Close the session. */
505
+ close(code?: number, reason?: string): void;
506
+ /** The underlying socket (escape hatch for advanced use). */
507
+ get socket(): WebSocketLike;
508
+ /** Current WebSocket readyState. */
509
+ get readyState(): number;
510
+ }
287
511
  /** Shape of `GET /v1/me`. Fields are best-effort / forward-compatible. */
288
512
  export interface MeResponse {
289
513
  object?: string;
@@ -744,6 +968,29 @@ export declare class PyAI {
744
968
  trigger: (callId: string, input: RecapCallTriggerInput) => Promise<RecapCallSummary>;
745
969
  };
746
970
  };
971
+ omni: {
972
+ /**
973
+ * Mint an ephemeral, origin-locked Omni session token a browser can use to
974
+ * open ONE realtime session **directly** — the public/private split for
975
+ * realtime. **Call this from your server** with a secret key holding
976
+ * `omni:session`; never ship the secret key to a page. Hand the returned
977
+ * `token` to the browser, which connects with
978
+ * `new WebSocket(session.url, ["pyai-key." + session.token])`. The token
979
+ * expires after `ttlSeconds` (default 60s) and only works from
980
+ * `allowedOrigins`. Scope `omni:session`.
981
+ */
982
+ createSession: (params: OmniSessionParams) => Promise<OmniSession>;
983
+ /**
984
+ * Open a live Omni agentic-voice session over `/v1/omni`. Returns an
985
+ * {@link OmniConnection} that handles the wire protocol's frame-key
986
+ * asymmetry for you — it sends `type`-keyed control frames (`configure`,
987
+ * `dtmf`) and parses `event`-keyed server frames — so you can't trip the #1
988
+ * Omni integration bug. Omni is zero-state: nothing to create first; the
989
+ * agent's behavior travels in the `configure` frame. Pass `token` (from
990
+ * `createSession`) to connect from a browser without the secret key.
991
+ */
992
+ connect: (opts?: OmniConnectOptions) => OmniConnection;
993
+ };
747
994
  /** Build the realtime WebSocket URL for the chosen product. */
748
995
  realtimeURL(opts?: RealtimeOptions): string;
749
996
  /** The subprotocol that carries the key on a WS upgrade (browser-safe auth). */
package/dist/index.js CHANGED
@@ -45,6 +45,8 @@ export const HearFrameType = {
45
45
  SpeechFinal: "speech_final",
46
46
  /** Corrected, full-context transcript following `speech_final`. */
47
47
  Final: "final",
48
+ /** Final billed-usage summary, emitted just before a graceful close. */
49
+ Usage: "usage",
48
50
  /** Server-side fault frame. */
49
51
  Error: "error",
50
52
  };
@@ -99,7 +101,14 @@ export class HearStream {
99
101
  this.ws.onopen = () => {
100
102
  if (opts.grounding) {
101
103
  try {
102
- this.ws.send(JSON.stringify({ type: "config", grounding: true }));
104
+ const cfg = { type: "config", grounding: true };
105
+ if (opts.groundingK != null)
106
+ cfg.grounding_k = opts.groundingK;
107
+ if (opts.groundingMinScore != null)
108
+ cfg.grounding_min_score = opts.groundingMinScore;
109
+ if (opts.groundingTimeoutMs != null)
110
+ cfg.grounding_timeout_ms = opts.groundingTimeoutMs;
111
+ this.ws.send(JSON.stringify(cfg));
103
112
  }
104
113
  catch {
105
114
  /* surfaced via onerror */
@@ -135,6 +144,9 @@ export class HearStream {
135
144
  case HearFrameType.Final:
136
145
  this.opts.onFinal?.(frame);
137
146
  break;
147
+ case HearFrameType.Usage:
148
+ this.opts.onUsage?.(frame);
149
+ break;
138
150
  case HearFrameType.Error:
139
151
  this.opts.onError?.(frame);
140
152
  break;
@@ -165,6 +177,164 @@ export class HearStream {
165
177
  return this.ws.readyState;
166
178
  }
167
179
  }
180
+ /* ------------------------------------------------------------------------- *
181
+ * Omni realtime (agentic voice) — typed client over the /v1/omni WebSocket
182
+ * ------------------------------------------------------------------------- */
183
+ /**
184
+ * Event names on Omni **server → client** frames. ⚠️ Note the asymmetry:
185
+ * inbound frames are keyed on `event`, but your **outbound** control frames
186
+ * (`configure`, `dtmf`, …) are keyed on `type`. {@link OmniConnection} handles
187
+ * both sides for you; this map is for matching frames in `onEvent`.
188
+ */
189
+ export const OmniEvent = {
190
+ /** Handshake; advertises protocol version + audio formats. */
191
+ Hello: "hello",
192
+ /** Ack for your `configure` frame (echoes the resolved `voice_id`). */
193
+ Configured: "configured",
194
+ /** Session is live; includes the resolved agent + audio caps. */
195
+ SessionStarted: "session_started",
196
+ /** Turn boundary (user/assistant speaking). */
197
+ Turn: "turn",
198
+ /** Incremental/final transcript text. */
199
+ Transcript: "transcript",
200
+ /** User interrupted; assistant audio is being cut. */
201
+ BargeIn: "barge_in",
202
+ /** Alias for barge-in on some engine builds. */
203
+ Flush: "flush",
204
+ /** Session is closing; see close code. */
205
+ SessionEnd: "session_end",
206
+ /** Server fault frame. */
207
+ Error: "error",
208
+ };
209
+ /**
210
+ * A live Omni agentic-voice session over `/v1/omni`. Hides the wire protocol's
211
+ * frame-key asymmetry: it sends control frames keyed on `type` (`configure`,
212
+ * `dtmf`) and parses server frames keyed on `event`, so you cannot trip the #1
213
+ * Omni integration bug (mirroring the server's `event` key on outbound, which
214
+ * is silently dropped). Construct via `pyai.omni.connect()`.
215
+ *
216
+ * @example
217
+ * const omni = pyai.omni.connect({
218
+ * rate: 16000,
219
+ * configure: { voice_id: "stock_emma_en_gb", persona: "You are a receptionist." },
220
+ * onAudio: (chunk) => speaker.write(chunk),
221
+ * onTranscript: (f) => console.log(f.text),
222
+ * });
223
+ * omni.sendAudio(pcm16Chunk); // stream caller audio continuously
224
+ */
225
+ export class OmniConnection {
226
+ ws;
227
+ opts;
228
+ closed = false;
229
+ constructor(url, subprotocol, opts) {
230
+ this.opts = opts;
231
+ const WS = opts.webSocket ?? globalThis.WebSocket;
232
+ if (!WS) {
233
+ throw new Error("No global WebSocket available; pass options.webSocket (e.g. the `ws` package) to omni.connect()");
234
+ }
235
+ this.ws = new WS(url, [subprotocol]);
236
+ this.ws.onopen = () => {
237
+ if (opts.configure) {
238
+ try {
239
+ this.configure(opts.configure);
240
+ }
241
+ catch {
242
+ /* surfaced via onerror */
243
+ }
244
+ }
245
+ opts.onOpen?.();
246
+ };
247
+ this.ws.onmessage = (ev) => this.handleMessage(ev.data);
248
+ this.ws.onerror = (ev) => opts.onError?.(ev instanceof Error ? ev : new Error("WebSocket error"));
249
+ this.ws.onclose = (ev) => {
250
+ this.closed = true;
251
+ opts.onClose?.(ev.code, ev.reason);
252
+ };
253
+ }
254
+ handleMessage(data) {
255
+ // Binary = agent audio (play it out). JSON text = an `event`-keyed frame.
256
+ if (typeof data !== "string") {
257
+ this.opts.onAudio?.(data);
258
+ return;
259
+ }
260
+ let frame;
261
+ try {
262
+ frame = JSON.parse(data);
263
+ }
264
+ catch {
265
+ this.opts.onError?.(new Error(`Unparseable Omni frame: ${data.slice(0, 120)}`));
266
+ return;
267
+ }
268
+ this.opts.onEvent?.(frame);
269
+ switch (frame.event) {
270
+ case OmniEvent.Hello:
271
+ this.opts.onHello?.(frame);
272
+ break;
273
+ case OmniEvent.Configured:
274
+ this.opts.onConfigured?.(frame);
275
+ break;
276
+ case OmniEvent.SessionStarted:
277
+ this.opts.onSessionStarted?.(frame);
278
+ break;
279
+ case OmniEvent.Turn:
280
+ this.opts.onTurn?.(frame);
281
+ break;
282
+ case OmniEvent.Transcript:
283
+ this.opts.onTranscript?.(frame);
284
+ break;
285
+ case OmniEvent.BargeIn:
286
+ case OmniEvent.Flush:
287
+ this.opts.onBargeIn?.(frame);
288
+ break;
289
+ case OmniEvent.SessionEnd:
290
+ this.opts.onSessionEnd?.(frame);
291
+ break;
292
+ case OmniEvent.Error:
293
+ this.opts.onError?.(frame);
294
+ break;
295
+ default:
296
+ // Unknown/forward-compatible frame — already delivered via onEvent.
297
+ break;
298
+ }
299
+ }
300
+ /**
301
+ * Send (or update) the agent config. Always emitted as
302
+ * `{"type":"configure", ...}` — the correct key. (A hand-rolled
303
+ * `{"event":"configure"}` is acked but silently dropped by the engine.)
304
+ */
305
+ configure(cfg) {
306
+ this.ws.send(JSON.stringify({ type: "configure", ...cfg }));
307
+ }
308
+ /** Stream a chunk of caller audio (PCM16 LE mono at the negotiated rate). */
309
+ sendAudio(chunk) {
310
+ this.ws.send(chunk);
311
+ }
312
+ /** Send DTMF digits as a `{"type":"dtmf"}` control frame. */
313
+ sendDtmf(digits) {
314
+ this.ws.send(JSON.stringify({ type: "dtmf", digits }));
315
+ }
316
+ /**
317
+ * Send an arbitrary control frame for forward-compat control types the SDK
318
+ * does not model yet. Reminder: client → server frames are keyed on `type`,
319
+ * never `event`.
320
+ */
321
+ send(frame) {
322
+ this.ws.send(JSON.stringify(frame));
323
+ }
324
+ /** Close the session. */
325
+ close(code = WSCloseCode.Normal, reason = "") {
326
+ if (!this.closed)
327
+ this.ws.close(code, reason);
328
+ }
329
+ /** The underlying socket (escape hatch for advanced use). */
330
+ get socket() {
331
+ return this.ws;
332
+ }
333
+ /** Current WebSocket readyState. */
334
+ get readyState() {
335
+ return this.ws.readyState;
336
+ }
337
+ }
168
338
  const RETRYABLE = new Set([429, 500, 502, 503, 504]);
169
339
  export class PyAI {
170
340
  apiKey;
@@ -510,17 +680,57 @@ export class PyAI {
510
680
  trigger: (callId, input) => this.postJson(`/v1/recap/calls/${encodeURIComponent(callId)}`, input),
511
681
  },
512
682
  };
683
+ // --- omni (agentic voice) ----------------------------------------------
684
+ omni = {
685
+ /**
686
+ * Mint an ephemeral, origin-locked Omni session token a browser can use to
687
+ * open ONE realtime session **directly** — the public/private split for
688
+ * realtime. **Call this from your server** with a secret key holding
689
+ * `omni:session`; never ship the secret key to a page. Hand the returned
690
+ * `token` to the browser, which connects with
691
+ * `new WebSocket(session.url, ["pyai-key." + session.token])`. The token
692
+ * expires after `ttlSeconds` (default 60s) and only works from
693
+ * `allowedOrigins`. Scope `omni:session`.
694
+ */
695
+ createSession: (params) => this.postJson("/v1/omni/sessions", {
696
+ allowed_origins: params.allowedOrigins,
697
+ ...(params.ttlSeconds !== undefined ? { ttl_seconds: params.ttlSeconds } : {}),
698
+ ...(params.sessionLabel !== undefined ? { session_label: params.sessionLabel } : {}),
699
+ }),
700
+ /**
701
+ * Open a live Omni agentic-voice session over `/v1/omni`. Returns an
702
+ * {@link OmniConnection} that handles the wire protocol's frame-key
703
+ * asymmetry for you — it sends `type`-keyed control frames (`configure`,
704
+ * `dtmf`) and parses `event`-keyed server frames — so you can't trip the #1
705
+ * Omni integration bug. Omni is zero-state: nothing to create first; the
706
+ * agent's behavior travels in the `configure` frame. Pass `token` (from
707
+ * `createSession`) to connect from a browser without the secret key.
708
+ */
709
+ connect: (opts = {}) => {
710
+ const query = { ...(opts.query ?? {}) };
711
+ if (opts.format)
712
+ query.format = opts.format;
713
+ if (opts.rate)
714
+ query.rate = String(opts.rate);
715
+ const url = this.realtimeURL({ product: "omni", sessionLabel: opts.sessionLabel, query });
716
+ const sub = opts.token ? `pyai-key.${opts.token}` : this.realtimeSubprotocol();
717
+ return new OmniConnection(url, sub, opts);
718
+ },
719
+ };
513
720
  // --- realtime (WebSocket) ----------------------------------------------
514
721
  /** Build the realtime WebSocket URL for the chosen product. */
515
722
  realtimeURL(opts = {}) {
516
723
  const wsBase = this.baseURL.replace(/^http/, "ws");
517
724
  const q = new URLSearchParams(opts.query ?? {});
518
725
  if ((opts.product ?? "omni") === "omni") {
519
- // Omni's native realtime surface is /v1/omni. agentId is an opaque label
520
- // authorized by the key's org. format/rate are load-bearing on the
521
- // connect URL, so default to browser-grade PCM16/24kHz.
522
- if (opts.agentId)
523
- q.set("agent_id", opts.agentId);
726
+ // Omni's native realtime surface is /v1/omni. The session is authorized by
727
+ // the key's org (zero-state) sessionLabel is an optional opaque tag.
728
+ // format/rate are load-bearing on the connect URL, so default to
729
+ // browser-grade PCM16/24kHz.
730
+ if (opts.sessionLabel)
731
+ q.set("session_label", opts.sessionLabel);
732
+ else if (opts.agentId)
733
+ q.set("agent_id", opts.agentId); // deprecated alias
524
734
  if (!q.has("format"))
525
735
  q.set("format", "pcm16");
526
736
  if (!q.has("rate"))
@@ -549,6 +759,10 @@ export class PyAI {
549
759
  q.set("encoding", opts.encoding);
550
760
  if (opts.interimResults !== undefined)
551
761
  q.set("interim_results", String(opts.interimResults));
762
+ if (opts.numerals !== undefined)
763
+ q.set("numerals", String(opts.numerals));
764
+ if (opts.endpointingMs !== undefined)
765
+ q.set("endpointing_ms", String(opts.endpointingMs));
552
766
  const qs = q.toString();
553
767
  return `${wsBase}/v1/audio/transcriptions/stream${qs ? `?${qs}` : ""}`;
554
768
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@pyai/sdk",
3
- "version": "0.2.0",
3
+ "version": "0.2.1",
4
4
  "description": "Official TypeScript/JavaScript SDK for PyAI — speech-to-text (Hear), text-to-speech (Speak), realtime voice agents (Omni), and call compliance (Trace).",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",
@@ -22,6 +22,7 @@
22
22
  },
23
23
  "homepage": "https://pyai.com",
24
24
  "bugs": "https://github.com/atomsai/pyai-platform-backend/issues",
25
+ "author": "PyAI",
25
26
  "scripts": {
26
27
  "build": "tsc -p tsconfig.build.json",
27
28
  "test": "node --test test/**/*.test.ts",
package/src/index.ts CHANGED
@@ -139,14 +139,55 @@ export interface CreateJobParams {
139
139
  }
140
140
 
141
141
  export interface RealtimeOptions {
142
- /** "omni" (agentic, needs agentId) or "flow" (voice duplex). Default "omni". */
142
+ /** "omni" (agentic voice) or "flow" (legacy voice duplex). Default "omni". */
143
143
  product?: "omni" | "flow";
144
- /** Required for omni: the agent to drive. */
144
+ /**
145
+ * Optional opaque tag echoed to your `kb_endpoint` and recorded on the call.
146
+ * Omni is zero-state: the session is authorized by the key's org, so there is
147
+ * nothing to create first and this tag is never required.
148
+ */
149
+ sessionLabel?: string;
150
+ /**
151
+ * @deprecated Use {@link sessionLabel}. Kept for back-compat — emitted as the
152
+ * `agent_id` query alias, which the gateway still accepts. Ignored if
153
+ * `sessionLabel` is set.
154
+ */
145
155
  agentId?: string;
146
156
  /** Extra query params (e.g. format, rate). */
147
157
  query?: Record<string, string>;
148
158
  }
149
159
 
160
+ /** Parameters for minting an ephemeral browser Omni session token (server-side). */
161
+ export interface OmniSessionParams {
162
+ /**
163
+ * Browser origins (scheme://host[:port]) the minted token may connect from.
164
+ * Required and non-empty — a browser token must be origin-locked. `*` is not
165
+ * allowed.
166
+ */
167
+ allowedOrigins: string[];
168
+ /** Token lifetime in seconds. Default 60 server-side; max 600. Keep it short. */
169
+ ttlSeconds?: number;
170
+ /** Optional opaque tag echoed back to your `kb_endpoint` and recorded on the call. */
171
+ sessionLabel?: string;
172
+ }
173
+
174
+ /**
175
+ * The result of {@link PyAI.omni}.createSession — a short-lived credential safe
176
+ * to hand to the browser. Use {@link OmniSession.token} as the WebSocket
177
+ * subprotocol `pyai-key.<token>` against {@link OmniSession.url}.
178
+ */
179
+ export interface OmniSession {
180
+ object?: "omni.session";
181
+ /** The ephemeral session token. Short-lived + origin-locked; browser-safe. */
182
+ token: string;
183
+ /** Token expiry, Unix epoch milliseconds. */
184
+ expires_at: number;
185
+ /** The Omni realtime WebSocket URL to connect to. */
186
+ url: string;
187
+ /** Echoed back when supplied on the request. */
188
+ session_label?: string;
189
+ }
190
+
150
191
  /* ------------------------------------------------------------------------- *
151
192
  * Stable enums — mirror the server so callers branch on named constants, not
152
193
  * magic strings, and a contract change surfaces in one place. These are plain
@@ -164,6 +205,8 @@ export const HearFrameType = {
164
205
  SpeechFinal: "speech_final",
165
206
  /** Corrected, full-context transcript following `speech_final`. */
166
207
  Final: "final",
208
+ /** Final billed-usage summary, emitted just before a graceful close. */
209
+ Usage: "usage",
167
210
  /** Server-side fault frame. */
168
211
  Error: "error",
169
212
  } as const;
@@ -239,6 +282,22 @@ export interface HearFinalFrame {
239
282
  grounding?: HearGroundingPassage[];
240
283
  }
241
284
 
285
+ /** Final billed-usage summary (`usage`), emitted just before a graceful close
286
+ * so realtime spend can be reconciled in-band (a realtime WS carries no
287
+ * `x-pyai-units` response header). Best-effort: absent if there was no billable
288
+ * audio or the close was abnormal. */
289
+ export interface HearUsageFrame {
290
+ type: "usage";
291
+ /** `hear` for plain streaming, `cue` when grounding was enabled. */
292
+ product: "hear" | "cue";
293
+ /** The billed meter (`hear.requests` or `cue.minutes`). */
294
+ meter: string;
295
+ /** Summed active-speech audio billed for the session, in seconds. */
296
+ audio_seconds: number;
297
+ /** The same quantity in minutes. */
298
+ minutes: number;
299
+ }
300
+
242
301
  /** Server fault frame (`error`). */
243
302
  export interface HearErrorFrame {
244
303
  type: "error";
@@ -246,7 +305,7 @@ export interface HearErrorFrame {
246
305
  message: string;
247
306
  }
248
307
 
249
- export type HearFrame = HearPartialFrame | HearFinalFrame | HearErrorFrame;
308
+ export type HearFrame = HearPartialFrame | HearFinalFrame | HearUsageFrame | HearErrorFrame;
250
309
 
251
310
  /**
252
311
  * Minimal structural WebSocket — matches both the browser/Node global
@@ -275,12 +334,32 @@ export interface HearStreamOptions {
275
334
  encoding?: "pcm16" | "opus";
276
335
  /** Emit eager partial hypotheses. Default true server-side. */
277
336
  interimResults?: boolean;
337
+ /**
338
+ * Format spoken numbers as digits in the transcript (e.g. "one two three" →
339
+ * "123"). Useful for voice agents that read back phone numbers, codes, and
340
+ * amounts. Default false (spoken form). Forwards `?numerals=true` on the URL.
341
+ */
342
+ numerals?: boolean;
343
+ /**
344
+ * Turn-segmentation tuning: trailing-pause (ms, 50–2000) that ends an
345
+ * utterance. Forwards `?endpointing_ms=` — clamped + honored once the engine
346
+ * supports it; a no-op when omitted. Drive end-of-turn yourself with
347
+ * {@link HearStream.commit} for full control today.
348
+ */
349
+ endpointingMs?: number;
278
350
  /**
279
351
  * Enable Cue knowledge-base grounding: sends `{type:"config",grounding:true}`
280
352
  * on open, after which `speech_final`/`final` frames carry a `grounding`
281
353
  * array. Bills a single `cue.minutes` line instead of the Hear rate.
282
354
  */
283
355
  grounding?: boolean;
356
+ /** Cue: number of KB passages to retrieve per turn (1–20, default 3). */
357
+ groundingK?: number;
358
+ /** Cue: drop passages scoring below this (0–1, default 0 = keep all). */
359
+ groundingMinScore?: number;
360
+ /** Cue: max ms to wait for retrieval at the final before failing open to
361
+ * `grounding: []` (50–2000, default 450). */
362
+ groundingTimeoutMs?: number;
284
363
  /** Extra query params merged onto the connect URL. */
285
364
  query?: Record<string, string>;
286
365
  /** Fired once the socket opens (after the optional grounding config). */
@@ -289,6 +368,8 @@ export interface HearStreamOptions {
289
368
  onPartial?: (frame: HearPartialFrame) => void;
290
369
  /** Fired on `speech_final` / `final`. */
291
370
  onFinal?: (frame: HearFinalFrame) => void;
371
+ /** Fired on the final `usage` summary (in-band realtime reconciliation). */
372
+ onUsage?: (frame: HearUsageFrame) => void;
292
373
  /** Fired on an `error` frame or a transport-level error. */
293
374
  onError?: (err: HearErrorFrame | Error) => void;
294
375
  /** Fired when the socket closes (code per `WSCloseCode`). */
@@ -320,7 +401,11 @@ export class HearStream {
320
401
  this.ws.onopen = () => {
321
402
  if (opts.grounding) {
322
403
  try {
323
- this.ws.send(JSON.stringify({ type: "config", grounding: true }));
404
+ const cfg: Record<string, unknown> = { type: "config", grounding: true };
405
+ if (opts.groundingK != null) cfg.grounding_k = opts.groundingK;
406
+ if (opts.groundingMinScore != null) cfg.grounding_min_score = opts.groundingMinScore;
407
+ if (opts.groundingTimeoutMs != null) cfg.grounding_timeout_ms = opts.groundingTimeoutMs;
408
+ this.ws.send(JSON.stringify(cfg));
324
409
  } catch {
325
410
  /* surfaced via onerror */
326
411
  }
@@ -354,6 +439,9 @@ export class HearStream {
354
439
  case HearFrameType.Final:
355
440
  this.opts.onFinal?.(frame);
356
441
  break;
442
+ case HearFrameType.Usage:
443
+ this.opts.onUsage?.(frame);
444
+ break;
357
445
  case HearFrameType.Error:
358
446
  this.opts.onError?.(frame);
359
447
  break;
@@ -389,6 +477,254 @@ export class HearStream {
389
477
  }
390
478
  }
391
479
 
480
+ /* ------------------------------------------------------------------------- *
481
+ * Omni realtime (agentic voice) — typed client over the /v1/omni WebSocket
482
+ * ------------------------------------------------------------------------- */
483
+
484
+ /**
485
+ * Event names on Omni **server → client** frames. ⚠️ Note the asymmetry:
486
+ * inbound frames are keyed on `event`, but your **outbound** control frames
487
+ * (`configure`, `dtmf`, …) are keyed on `type`. {@link OmniConnection} handles
488
+ * both sides for you; this map is for matching frames in `onEvent`.
489
+ */
490
+ export const OmniEvent = {
491
+ /** Handshake; advertises protocol version + audio formats. */
492
+ Hello: "hello",
493
+ /** Ack for your `configure` frame (echoes the resolved `voice_id`). */
494
+ Configured: "configured",
495
+ /** Session is live; includes the resolved agent + audio caps. */
496
+ SessionStarted: "session_started",
497
+ /** Turn boundary (user/assistant speaking). */
498
+ Turn: "turn",
499
+ /** Incremental/final transcript text. */
500
+ Transcript: "transcript",
501
+ /** User interrupted; assistant audio is being cut. */
502
+ BargeIn: "barge_in",
503
+ /** Alias for barge-in on some engine builds. */
504
+ Flush: "flush",
505
+ /** Session is closing; see close code. */
506
+ SessionEnd: "session_end",
507
+ /** Server fault frame. */
508
+ Error: "error",
509
+ } as const;
510
+ export type OmniEvent = (typeof OmniEvent)[keyof typeof OmniEvent];
511
+
512
+ /** A server → client Omni frame. Keyed on `event` (not `type`); open for forward-compat. */
513
+ export interface OmniServerFrame {
514
+ event: string;
515
+ [k: string]: unknown;
516
+ }
517
+
518
+ /** A binary agent-audio chunk delivered to {@link OmniConnectOptions.onAudio}. */
519
+ export type OmniAudioChunk = ArrayBuffer | ArrayBufferView | Blob;
520
+
521
+ /**
522
+ * The agent config sent as the `configure` control frame. These are the wire
523
+ * (snake_case) fields the engine reads; it's an **open bag**, so forward-compat
524
+ * fields (e.g. `greeting` / `language` once the engine acks them) pass straight
525
+ * through. The SDK supplies the `{"type":"configure"}` envelope for you — which
526
+ * is the whole point: a hand-rolled `{"event":"configure"}` is acked but
527
+ * silently dropped, leaving the agent with no brain and zero turns.
528
+ */
529
+ export interface OmniConfigure {
530
+ /** Voice to speak with (stock / clone / designed id). */
531
+ voice_id?: string;
532
+ /** System prompt / role + instructions for the agent. */
533
+ persona?: string;
534
+ /** Customer-hosted URL the engine calls per turn for grounding. */
535
+ kb_endpoint?: string;
536
+ /** Bearer the engine presents to `kb_endpoint`. */
537
+ kb_token?: string;
538
+ /** Forward-compatible: any other key the engine honors. */
539
+ [k: string]: unknown;
540
+ }
541
+
542
+ export interface OmniConnectOptions {
543
+ /** Optional opaque per-session tag echoed to your `kb_endpoint`. Omni is zero-state. */
544
+ sessionLabel?: string;
545
+ /**
546
+ * Ephemeral browser token from `omni.createSession`. When set it is used as
547
+ * the WS subprotocol (`pyai-key.<token>`) instead of the client's secret key,
548
+ * so a page never holds a secret key.
549
+ */
550
+ token?: string;
551
+ /** Connect-URL sample rate: 24000 browser, 16000 wideband telephony, 8000 G.711/Twilio. */
552
+ rate?: 24000 | 16000 | 8000;
553
+ /** Connect-URL audio format. Default `pcm16`. */
554
+ format?: "pcm16";
555
+ /**
556
+ * Agent config sent as `{"type":"configure",...}` the instant the socket
557
+ * opens. Omit to send it yourself later via {@link OmniConnection.configure}.
558
+ */
559
+ configure?: OmniConfigure;
560
+ /** Extra query params merged onto the connect URL. */
561
+ query?: Record<string, string>;
562
+ /** Fired once the socket opens (after the optional auto-configure). */
563
+ onOpen?: () => void;
564
+ /** Fired for each binary agent-audio chunk — play it out as it arrives. */
565
+ onAudio?: (chunk: OmniAudioChunk) => void;
566
+ /** Fired on the `hello` handshake frame. */
567
+ onHello?: (frame: OmniServerFrame) => void;
568
+ /** Fired on the `configured` ack. */
569
+ onConfigured?: (frame: OmniServerFrame) => void;
570
+ /** Fired on `session_started`. */
571
+ onSessionStarted?: (frame: OmniServerFrame) => void;
572
+ /** Fired on `turn` boundaries. */
573
+ onTurn?: (frame: OmniServerFrame) => void;
574
+ /** Fired on `transcript` text frames. */
575
+ onTranscript?: (frame: OmniServerFrame) => void;
576
+ /** Fired on `barge_in` / `flush` (user interrupted). */
577
+ onBargeIn?: (frame: OmniServerFrame) => void;
578
+ /** Fired on `session_end`. */
579
+ onSessionEnd?: (frame: OmniServerFrame) => void;
580
+ /** Fired on EVERY JSON frame (including unknown/forward-compat ones). */
581
+ onEvent?: (frame: OmniServerFrame) => void;
582
+ /** Fired on an `error` frame or a transport-level error. */
583
+ onError?: (err: OmniServerFrame | Error) => void;
584
+ /** Fired when the socket closes (code per {@link WSCloseCode}). */
585
+ onClose?: (code: number, reason: string) => void;
586
+ /** Injectable WebSocket constructor (defaults to the global). */
587
+ webSocket?: WebSocketCtor;
588
+ }
589
+
590
+ /**
591
+ * A live Omni agentic-voice session over `/v1/omni`. Hides the wire protocol's
592
+ * frame-key asymmetry: it sends control frames keyed on `type` (`configure`,
593
+ * `dtmf`) and parses server frames keyed on `event`, so you cannot trip the #1
594
+ * Omni integration bug (mirroring the server's `event` key on outbound, which
595
+ * is silently dropped). Construct via `pyai.omni.connect()`.
596
+ *
597
+ * @example
598
+ * const omni = pyai.omni.connect({
599
+ * rate: 16000,
600
+ * configure: { voice_id: "stock_emma_en_gb", persona: "You are a receptionist." },
601
+ * onAudio: (chunk) => speaker.write(chunk),
602
+ * onTranscript: (f) => console.log(f.text),
603
+ * });
604
+ * omni.sendAudio(pcm16Chunk); // stream caller audio continuously
605
+ */
606
+ export class OmniConnection {
607
+ private readonly ws: WebSocketLike;
608
+ private readonly opts: OmniConnectOptions;
609
+ private closed = false;
610
+
611
+ constructor(url: string, subprotocol: string, opts: OmniConnectOptions) {
612
+ this.opts = opts;
613
+ const WS = opts.webSocket ?? (globalThis as { WebSocket?: WebSocketCtor }).WebSocket;
614
+ if (!WS) {
615
+ throw new Error(
616
+ "No global WebSocket available; pass options.webSocket (e.g. the `ws` package) to omni.connect()",
617
+ );
618
+ }
619
+ this.ws = new WS(url, [subprotocol]);
620
+ this.ws.onopen = () => {
621
+ if (opts.configure) {
622
+ try {
623
+ this.configure(opts.configure);
624
+ } catch {
625
+ /* surfaced via onerror */
626
+ }
627
+ }
628
+ opts.onOpen?.();
629
+ };
630
+ this.ws.onmessage = (ev) => this.handleMessage(ev.data);
631
+ this.ws.onerror = (ev) => opts.onError?.(ev instanceof Error ? ev : new Error("WebSocket error"));
632
+ this.ws.onclose = (ev) => {
633
+ this.closed = true;
634
+ opts.onClose?.(ev.code, ev.reason);
635
+ };
636
+ }
637
+
638
+ private handleMessage(data: unknown): void {
639
+ // Binary = agent audio (play it out). JSON text = an `event`-keyed frame.
640
+ if (typeof data !== "string") {
641
+ this.opts.onAudio?.(data as OmniAudioChunk);
642
+ return;
643
+ }
644
+ let frame: OmniServerFrame;
645
+ try {
646
+ frame = JSON.parse(data) as OmniServerFrame;
647
+ } catch {
648
+ this.opts.onError?.(new Error(`Unparseable Omni frame: ${data.slice(0, 120)}`));
649
+ return;
650
+ }
651
+ this.opts.onEvent?.(frame);
652
+ switch (frame.event) {
653
+ case OmniEvent.Hello:
654
+ this.opts.onHello?.(frame);
655
+ break;
656
+ case OmniEvent.Configured:
657
+ this.opts.onConfigured?.(frame);
658
+ break;
659
+ case OmniEvent.SessionStarted:
660
+ this.opts.onSessionStarted?.(frame);
661
+ break;
662
+ case OmniEvent.Turn:
663
+ this.opts.onTurn?.(frame);
664
+ break;
665
+ case OmniEvent.Transcript:
666
+ this.opts.onTranscript?.(frame);
667
+ break;
668
+ case OmniEvent.BargeIn:
669
+ case OmniEvent.Flush:
670
+ this.opts.onBargeIn?.(frame);
671
+ break;
672
+ case OmniEvent.SessionEnd:
673
+ this.opts.onSessionEnd?.(frame);
674
+ break;
675
+ case OmniEvent.Error:
676
+ this.opts.onError?.(frame);
677
+ break;
678
+ default:
679
+ // Unknown/forward-compatible frame — already delivered via onEvent.
680
+ break;
681
+ }
682
+ }
683
+
684
+ /**
685
+ * Send (or update) the agent config. Always emitted as
686
+ * `{"type":"configure", ...}` — the correct key. (A hand-rolled
687
+ * `{"event":"configure"}` is acked but silently dropped by the engine.)
688
+ */
689
+ configure(cfg: OmniConfigure): void {
690
+ this.ws.send(JSON.stringify({ type: "configure", ...cfg }));
691
+ }
692
+
693
+ /** Stream a chunk of caller audio (PCM16 LE mono at the negotiated rate). */
694
+ sendAudio(chunk: ArrayBufferLike | ArrayBufferView | Blob): void {
695
+ this.ws.send(chunk);
696
+ }
697
+
698
+ /** Send DTMF digits as a `{"type":"dtmf"}` control frame. */
699
+ sendDtmf(digits: string): void {
700
+ this.ws.send(JSON.stringify({ type: "dtmf", digits }));
701
+ }
702
+
703
+ /**
704
+ * Send an arbitrary control frame for forward-compat control types the SDK
705
+ * does not model yet. Reminder: client → server frames are keyed on `type`,
706
+ * never `event`.
707
+ */
708
+ send(frame: Record<string, unknown>): void {
709
+ this.ws.send(JSON.stringify(frame));
710
+ }
711
+
712
+ /** Close the session. */
713
+ close(code: number = WSCloseCode.Normal, reason = ""): void {
714
+ if (!this.closed) this.ws.close(code, reason);
715
+ }
716
+
717
+ /** The underlying socket (escape hatch for advanced use). */
718
+ get socket(): WebSocketLike {
719
+ return this.ws;
720
+ }
721
+
722
+ /** Current WebSocket readyState. */
723
+ get readyState(): number {
724
+ return this.ws.readyState;
725
+ }
726
+ }
727
+
392
728
  /* ------------------------------------------------------------------------- *
393
729
  * Key introspection — GET /v1/me
394
730
  * ------------------------------------------------------------------------- */
@@ -1064,6 +1400,45 @@ export class PyAI {
1064
1400
  },
1065
1401
  };
1066
1402
 
1403
+ // --- omni (agentic voice) ----------------------------------------------
1404
+
1405
+ omni = {
1406
+ /**
1407
+ * Mint an ephemeral, origin-locked Omni session token a browser can use to
1408
+ * open ONE realtime session **directly** — the public/private split for
1409
+ * realtime. **Call this from your server** with a secret key holding
1410
+ * `omni:session`; never ship the secret key to a page. Hand the returned
1411
+ * `token` to the browser, which connects with
1412
+ * `new WebSocket(session.url, ["pyai-key." + session.token])`. The token
1413
+ * expires after `ttlSeconds` (default 60s) and only works from
1414
+ * `allowedOrigins`. Scope `omni:session`.
1415
+ */
1416
+ createSession: (params: OmniSessionParams): Promise<OmniSession> =>
1417
+ this.postJson("/v1/omni/sessions", {
1418
+ allowed_origins: params.allowedOrigins,
1419
+ ...(params.ttlSeconds !== undefined ? { ttl_seconds: params.ttlSeconds } : {}),
1420
+ ...(params.sessionLabel !== undefined ? { session_label: params.sessionLabel } : {}),
1421
+ }),
1422
+
1423
+ /**
1424
+ * Open a live Omni agentic-voice session over `/v1/omni`. Returns an
1425
+ * {@link OmniConnection} that handles the wire protocol's frame-key
1426
+ * asymmetry for you — it sends `type`-keyed control frames (`configure`,
1427
+ * `dtmf`) and parses `event`-keyed server frames — so you can't trip the #1
1428
+ * Omni integration bug. Omni is zero-state: nothing to create first; the
1429
+ * agent's behavior travels in the `configure` frame. Pass `token` (from
1430
+ * `createSession`) to connect from a browser without the secret key.
1431
+ */
1432
+ connect: (opts: OmniConnectOptions = {}): OmniConnection => {
1433
+ const query: Record<string, string> = { ...(opts.query ?? {}) };
1434
+ if (opts.format) query.format = opts.format;
1435
+ if (opts.rate) query.rate = String(opts.rate);
1436
+ const url = this.realtimeURL({ product: "omni", sessionLabel: opts.sessionLabel, query });
1437
+ const sub = opts.token ? `pyai-key.${opts.token}` : this.realtimeSubprotocol();
1438
+ return new OmniConnection(url, sub, opts);
1439
+ },
1440
+ };
1441
+
1067
1442
  // --- realtime (WebSocket) ----------------------------------------------
1068
1443
 
1069
1444
  /** Build the realtime WebSocket URL for the chosen product. */
@@ -1071,10 +1446,12 @@ export class PyAI {
1071
1446
  const wsBase = this.baseURL.replace(/^http/, "ws");
1072
1447
  const q = new URLSearchParams(opts.query ?? {});
1073
1448
  if ((opts.product ?? "omni") === "omni") {
1074
- // Omni's native realtime surface is /v1/omni. agentId is an opaque label
1075
- // authorized by the key's org. format/rate are load-bearing on the
1076
- // connect URL, so default to browser-grade PCM16/24kHz.
1077
- if (opts.agentId) q.set("agent_id", opts.agentId);
1449
+ // Omni's native realtime surface is /v1/omni. The session is authorized by
1450
+ // the key's org (zero-state) sessionLabel is an optional opaque tag.
1451
+ // format/rate are load-bearing on the connect URL, so default to
1452
+ // browser-grade PCM16/24kHz.
1453
+ if (opts.sessionLabel) q.set("session_label", opts.sessionLabel);
1454
+ else if (opts.agentId) q.set("agent_id", opts.agentId); // deprecated alias
1078
1455
  if (!q.has("format")) q.set("format", "pcm16");
1079
1456
  if (!q.has("rate")) q.set("rate", "24000");
1080
1457
  const qs = q.toString();
@@ -1098,6 +1475,8 @@ export class PyAI {
1098
1475
  if (opts.sampleRate !== undefined) q.set("sample_rate", String(opts.sampleRate));
1099
1476
  if (opts.encoding) q.set("encoding", opts.encoding);
1100
1477
  if (opts.interimResults !== undefined) q.set("interim_results", String(opts.interimResults));
1478
+ if (opts.numerals !== undefined) q.set("numerals", String(opts.numerals));
1479
+ if (opts.endpointingMs !== undefined) q.set("endpointing_ms", String(opts.endpointingMs));
1101
1480
  const qs = q.toString();
1102
1481
  return `${wsBase}/v1/audio/transcriptions/stream${qs ? `?${qs}` : ""}`;
1103
1482
  }