@pyai/sdk 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -14
- package/dist/index.d.ts +250 -3
- package/dist/index.js +220 -6
- package/package.json +2 -1
- package/src/index.ts +387 -8
package/README.md
CHANGED
|
@@ -31,13 +31,13 @@ import PyAI from "@pyai/sdk";
|
|
|
31
31
|
const pyai = new PyAI({ apiKey: process.env.PYAI_API_KEY! });
|
|
32
32
|
|
|
33
33
|
// Text-to-speech
|
|
34
|
-
const audio = await pyai.audio.speech({ input: "Hello from PyAI.", voice: "
|
|
34
|
+
const audio = await pyai.audio.speech({ input: "Hello from PyAI.", voice: "stock_emma_en_gb" });
|
|
35
35
|
await Bun.write?.("hello.wav", audio); // or fs.writeFile in Node
|
|
36
36
|
|
|
37
37
|
// Text-to-speech, streamed — start playing/forwarding at the first chunk
|
|
38
38
|
// (tens of ms) instead of waiting for the whole clip. Use mp3 for smooth
|
|
39
39
|
// progressive playback.
|
|
40
|
-
const stream = await pyai.audio.speechStream({ input: "Hello from PyAI.", voice: "
|
|
40
|
+
const stream = await pyai.audio.speechStream({ input: "Hello from PyAI.", voice: "stock_emma_en_gb", response_format: "mp3" });
|
|
41
41
|
for await (const chunk of stream) writeToSpeakerOrResponse(chunk);
|
|
42
42
|
|
|
43
43
|
// Voices
|
|
@@ -53,20 +53,42 @@ const done = await pyai.transcriptionJobs.get(job.job_id);
|
|
|
53
53
|
|
|
54
54
|
## Realtime (Omni)
|
|
55
55
|
|
|
56
|
-
|
|
56
|
+
`omni.connect()` opens an agentic-voice session and hides the wire protocol —
|
|
57
|
+
including its **frame-key asymmetry** (your control frames are keyed on `type`,
|
|
58
|
+
the server's frames are keyed on `event`). It sends a `type`-keyed `configure`
|
|
59
|
+
the instant the socket opens and routes server frames to typed callbacks, so you
|
|
60
|
+
**can't** trip the #1 Omni integration bug (a hand-rolled `{"event":"configure"}`
|
|
61
|
+
is acked but silently dropped, giving you a connected session with zero turns):
|
|
57
62
|
|
|
58
63
|
```ts
|
|
59
|
-
|
|
60
|
-
|
|
64
|
+
// Omni is zero-state: the key's org authorizes the session — nothing to create.
|
|
65
|
+
const omni = pyai.omni.connect({
|
|
66
|
+
rate: 16000, // 24000 browser · 16000 wideband telephony · 8000 G.711/Twilio
|
|
67
|
+
configure: { voice_id: "stock_emma_en_gb", persona: "You are a receptionist." },
|
|
68
|
+
onAudio: (chunk) => speaker.write(chunk), // binary agent audio — play it out
|
|
69
|
+
onTranscript: (f) => console.log(f.text),
|
|
70
|
+
onError: (e) => console.error(e),
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
omni.sendAudio(pcm16Chunk); // stream caller audio continuously (server-side VAD)
|
|
74
|
+
omni.sendDtmf("5");
|
|
75
|
+
omni.close();
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
From the browser, mint an ephemeral token server-side with
|
|
79
|
+
`pyai.omni.createSession({ allowedOrigins })` and pass it as `token` so the page
|
|
80
|
+
never holds a secret key:
|
|
61
81
|
|
|
62
|
-
|
|
63
|
-
const
|
|
64
|
-
const proto = pyai.realtimeSubprotocol();
|
|
82
|
+
```ts
|
|
83
|
+
const omni = pyai.omni.connect({ token: session.token, configure: { voice_id, persona } });
|
|
65
84
|
```
|
|
66
85
|
|
|
67
|
-
> Omni uses the native `wss://api.pyai.com/v1/omni` surface
|
|
68
|
-
>
|
|
69
|
-
>
|
|
86
|
+
> Omni uses the native `wss://api.pyai.com/v1/omni` surface and is **zero-state**
|
|
87
|
+
> — no agent to create, `sessionLabel` is an optional opaque tag (never
|
|
88
|
+
> required). Need the raw socket? `pyai.realtimeURL({ product: "omni" })` +
|
|
89
|
+
> `pyai.realtimeSubprotocol()` (or `pyai.connectRealtime()`) still work;
|
|
90
|
+
> `product: "flow"` uses `/v1/realtime`. The older `/v2/omni/chat` URL and the
|
|
91
|
+
> `agentId` option are deprecated but still work.
|
|
70
92
|
|
|
71
93
|
## Streaming speech-to-text (Hear / Cue)
|
|
72
94
|
|
|
@@ -116,7 +138,7 @@ drop the hand-rolled resampler + μ-law encoder entirely:
|
|
|
116
138
|
// Twilio/SIP-ready in one param: raw 8 kHz mono μ-law, no client-side DSP.
|
|
117
139
|
const ulaw = await pyai.audio.speech({
|
|
118
140
|
input: "Your appointment is confirmed.",
|
|
119
|
-
voice: "
|
|
141
|
+
voice: "stock_emma_en_gb",
|
|
120
142
|
response_format: "g711_ulaw", // -> audio/basic, forced 8 kHz
|
|
121
143
|
});
|
|
122
144
|
// base64-encode `ulaw` straight into a Twilio media frame.
|
|
@@ -176,7 +198,7 @@ const quality = detail.quality_metrics; // { wer?, ttfb_ms?,
|
|
|
176
198
|
once the engine supports them and otherwise ignored — so it's always safe to send:
|
|
177
199
|
|
|
178
200
|
```ts
|
|
179
|
-
await pyai.audio.speech({ input: "Hello", voice: "
|
|
201
|
+
await pyai.audio.speech({ input: "Hello", voice: "stock_emma_en_gb", seed: 42, temperature: 0 });
|
|
180
202
|
await pyai.audio.transcriptions.create({ file: wavBlob, seed: 42 });
|
|
181
203
|
```
|
|
182
204
|
|
|
@@ -225,7 +247,7 @@ Other commands:
|
|
|
225
247
|
```bash
|
|
226
248
|
pyai models
|
|
227
249
|
pyai voices --gender female --region en_us
|
|
228
|
-
pyai speak --text "Hello" --voice
|
|
250
|
+
pyai speak --text "Hello" --voice stock_emma_en_gb --out hello.wav
|
|
229
251
|
pyai transcribe --url https://example.com/call.wav --diarize --poll
|
|
230
252
|
```
|
|
231
253
|
|
package/dist/index.d.ts
CHANGED
|
@@ -121,13 +121,52 @@ export interface CreateJobParams {
|
|
|
121
121
|
webhook_url?: string;
|
|
122
122
|
}
|
|
123
123
|
export interface RealtimeOptions {
|
|
124
|
-
/** "omni" (agentic
|
|
124
|
+
/** "omni" (agentic voice) or "flow" (legacy voice duplex). Default "omni". */
|
|
125
125
|
product?: "omni" | "flow";
|
|
126
|
-
/**
|
|
126
|
+
/**
|
|
127
|
+
* Optional opaque tag echoed to your `kb_endpoint` and recorded on the call.
|
|
128
|
+
* Omni is zero-state: the session is authorized by the key's org, so there is
|
|
129
|
+
* nothing to create first and this tag is never required.
|
|
130
|
+
*/
|
|
131
|
+
sessionLabel?: string;
|
|
132
|
+
/**
|
|
133
|
+
* @deprecated Use {@link sessionLabel}. Kept for back-compat — emitted as the
|
|
134
|
+
* `agent_id` query alias, which the gateway still accepts. Ignored if
|
|
135
|
+
* `sessionLabel` is set.
|
|
136
|
+
*/
|
|
127
137
|
agentId?: string;
|
|
128
138
|
/** Extra query params (e.g. format, rate). */
|
|
129
139
|
query?: Record<string, string>;
|
|
130
140
|
}
|
|
141
|
+
/** Parameters for minting an ephemeral browser Omni session token (server-side). */
|
|
142
|
+
export interface OmniSessionParams {
|
|
143
|
+
/**
|
|
144
|
+
* Browser origins (scheme://host[:port]) the minted token may connect from.
|
|
145
|
+
* Required and non-empty — a browser token must be origin-locked. `*` is not
|
|
146
|
+
* allowed.
|
|
147
|
+
*/
|
|
148
|
+
allowedOrigins: string[];
|
|
149
|
+
/** Token lifetime in seconds. Default 60 server-side; max 600. Keep it short. */
|
|
150
|
+
ttlSeconds?: number;
|
|
151
|
+
/** Optional opaque tag echoed back to your `kb_endpoint` and recorded on the call. */
|
|
152
|
+
sessionLabel?: string;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* The result of {@link PyAI.omni}.createSession — a short-lived credential safe
|
|
156
|
+
* to hand to the browser. Use {@link OmniSession.token} as the WebSocket
|
|
157
|
+
* subprotocol `pyai-key.<token>` against {@link OmniSession.url}.
|
|
158
|
+
*/
|
|
159
|
+
export interface OmniSession {
|
|
160
|
+
object?: "omni.session";
|
|
161
|
+
/** The ephemeral session token. Short-lived + origin-locked; browser-safe. */
|
|
162
|
+
token: string;
|
|
163
|
+
/** Token expiry, Unix epoch milliseconds. */
|
|
164
|
+
expires_at: number;
|
|
165
|
+
/** The Omni realtime WebSocket URL to connect to. */
|
|
166
|
+
url: string;
|
|
167
|
+
/** Echoed back when supplied on the request. */
|
|
168
|
+
session_label?: string;
|
|
169
|
+
}
|
|
131
170
|
/** Frame `type`s emitted by the Hear streaming-STT WebSocket. */
|
|
132
171
|
export declare const HearFrameType: {
|
|
133
172
|
/** Eager live hypothesis for the current utterance. */
|
|
@@ -138,6 +177,8 @@ export declare const HearFrameType: {
|
|
|
138
177
|
readonly SpeechFinal: "speech_final";
|
|
139
178
|
/** Corrected, full-context transcript following `speech_final`. */
|
|
140
179
|
readonly Final: "final";
|
|
180
|
+
/** Final billed-usage summary, emitted just before a graceful close. */
|
|
181
|
+
readonly Usage: "usage";
|
|
141
182
|
/** Server-side fault frame. */
|
|
142
183
|
readonly Error: "error";
|
|
143
184
|
};
|
|
@@ -203,13 +244,28 @@ export interface HearFinalFrame {
|
|
|
203
244
|
/** Present only with Cue grounding enabled (top KB passages). */
|
|
204
245
|
grounding?: HearGroundingPassage[];
|
|
205
246
|
}
|
|
247
|
+
/** Final billed-usage summary (`usage`), emitted just before a graceful close
|
|
248
|
+
* so realtime spend can be reconciled in-band (a realtime WS carries no
|
|
249
|
+
* `x-pyai-units` response header). Best-effort: absent if there was no billable
|
|
250
|
+
* audio or the close was abnormal. */
|
|
251
|
+
export interface HearUsageFrame {
|
|
252
|
+
type: "usage";
|
|
253
|
+
/** `hear` for plain streaming, `cue` when grounding was enabled. */
|
|
254
|
+
product: "hear" | "cue";
|
|
255
|
+
/** The billed meter (`hear.requests` or `cue.minutes`). */
|
|
256
|
+
meter: string;
|
|
257
|
+
/** Summed active-speech audio billed for the session, in seconds. */
|
|
258
|
+
audio_seconds: number;
|
|
259
|
+
/** The same quantity in minutes. */
|
|
260
|
+
minutes: number;
|
|
261
|
+
}
|
|
206
262
|
/** Server fault frame (`error`). */
|
|
207
263
|
export interface HearErrorFrame {
|
|
208
264
|
type: "error";
|
|
209
265
|
code?: string;
|
|
210
266
|
message: string;
|
|
211
267
|
}
|
|
212
|
-
export type HearFrame = HearPartialFrame | HearFinalFrame | HearErrorFrame;
|
|
268
|
+
export type HearFrame = HearPartialFrame | HearFinalFrame | HearUsageFrame | HearErrorFrame;
|
|
213
269
|
/**
|
|
214
270
|
* Minimal structural WebSocket — matches both the browser/Node global
|
|
215
271
|
* `WebSocket` and the `ws` package, and lets tests inject a mock.
|
|
@@ -240,12 +296,32 @@ export interface HearStreamOptions {
|
|
|
240
296
|
encoding?: "pcm16" | "opus";
|
|
241
297
|
/** Emit eager partial hypotheses. Default true server-side. */
|
|
242
298
|
interimResults?: boolean;
|
|
299
|
+
/**
|
|
300
|
+
* Format spoken numbers as digits in the transcript (e.g. "one two three" →
|
|
301
|
+
* "123"). Useful for voice agents that read back phone numbers, codes, and
|
|
302
|
+
* amounts. Default false (spoken form). Forwards `?numerals=true` on the URL.
|
|
303
|
+
*/
|
|
304
|
+
numerals?: boolean;
|
|
305
|
+
/**
|
|
306
|
+
* Turn-segmentation tuning: trailing-pause (ms, 50–2000) that ends an
|
|
307
|
+
* utterance. Forwards `?endpointing_ms=` — clamped + honored once the engine
|
|
308
|
+
* supports it; a no-op when omitted. Drive end-of-turn yourself with
|
|
309
|
+
* {@link HearStream.commit} for full control today.
|
|
310
|
+
*/
|
|
311
|
+
endpointingMs?: number;
|
|
243
312
|
/**
|
|
244
313
|
* Enable Cue knowledge-base grounding: sends `{type:"config",grounding:true}`
|
|
245
314
|
* on open, after which `speech_final`/`final` frames carry a `grounding`
|
|
246
315
|
* array. Bills a single `cue.minutes` line instead of the Hear rate.
|
|
247
316
|
*/
|
|
248
317
|
grounding?: boolean;
|
|
318
|
+
/** Cue: number of KB passages to retrieve per turn (1–20, default 3). */
|
|
319
|
+
groundingK?: number;
|
|
320
|
+
/** Cue: drop passages scoring below this (0–1, default 0 = keep all). */
|
|
321
|
+
groundingMinScore?: number;
|
|
322
|
+
/** Cue: max ms to wait for retrieval at the final before failing open to
|
|
323
|
+
* `grounding: []` (50–2000, default 450). */
|
|
324
|
+
groundingTimeoutMs?: number;
|
|
249
325
|
/** Extra query params merged onto the connect URL. */
|
|
250
326
|
query?: Record<string, string>;
|
|
251
327
|
/** Fired once the socket opens (after the optional grounding config). */
|
|
@@ -254,6 +330,8 @@ export interface HearStreamOptions {
|
|
|
254
330
|
onPartial?: (frame: HearPartialFrame) => void;
|
|
255
331
|
/** Fired on `speech_final` / `final`. */
|
|
256
332
|
onFinal?: (frame: HearFinalFrame) => void;
|
|
333
|
+
/** Fired on the final `usage` summary (in-band realtime reconciliation). */
|
|
334
|
+
onUsage?: (frame: HearUsageFrame) => void;
|
|
257
335
|
/** Fired on an `error` frame or a transport-level error. */
|
|
258
336
|
onError?: (err: HearErrorFrame | Error) => void;
|
|
259
337
|
/** Fired when the socket closes (code per `WSCloseCode`). */
|
|
@@ -284,6 +362,152 @@ export declare class HearStream {
|
|
|
284
362
|
/** Current WebSocket readyState. */
|
|
285
363
|
get readyState(): number;
|
|
286
364
|
}
|
|
365
|
+
/**
|
|
366
|
+
* Event names on Omni **server → client** frames. ⚠️ Note the asymmetry:
|
|
367
|
+
* inbound frames are keyed on `event`, but your **outbound** control frames
|
|
368
|
+
* (`configure`, `dtmf`, …) are keyed on `type`. {@link OmniConnection} handles
|
|
369
|
+
* both sides for you; this map is for matching frames in `onEvent`.
|
|
370
|
+
*/
|
|
371
|
+
export declare const OmniEvent: {
|
|
372
|
+
/** Handshake; advertises protocol version + audio formats. */
|
|
373
|
+
readonly Hello: "hello";
|
|
374
|
+
/** Ack for your `configure` frame (echoes the resolved `voice_id`). */
|
|
375
|
+
readonly Configured: "configured";
|
|
376
|
+
/** Session is live; includes the resolved agent + audio caps. */
|
|
377
|
+
readonly SessionStarted: "session_started";
|
|
378
|
+
/** Turn boundary (user/assistant speaking). */
|
|
379
|
+
readonly Turn: "turn";
|
|
380
|
+
/** Incremental/final transcript text. */
|
|
381
|
+
readonly Transcript: "transcript";
|
|
382
|
+
/** User interrupted; assistant audio is being cut. */
|
|
383
|
+
readonly BargeIn: "barge_in";
|
|
384
|
+
/** Alias for barge-in on some engine builds. */
|
|
385
|
+
readonly Flush: "flush";
|
|
386
|
+
/** Session is closing; see close code. */
|
|
387
|
+
readonly SessionEnd: "session_end";
|
|
388
|
+
/** Server fault frame. */
|
|
389
|
+
readonly Error: "error";
|
|
390
|
+
};
|
|
391
|
+
export type OmniEvent = (typeof OmniEvent)[keyof typeof OmniEvent];
|
|
392
|
+
/** A server → client Omni frame. Keyed on `event` (not `type`); open for forward-compat. */
|
|
393
|
+
export interface OmniServerFrame {
|
|
394
|
+
event: string;
|
|
395
|
+
[k: string]: unknown;
|
|
396
|
+
}
|
|
397
|
+
/** A binary agent-audio chunk delivered to {@link OmniConnectOptions.onAudio}. */
|
|
398
|
+
export type OmniAudioChunk = ArrayBuffer | ArrayBufferView | Blob;
|
|
399
|
+
/**
|
|
400
|
+
* The agent config sent as the `configure` control frame. These are the wire
|
|
401
|
+
* (snake_case) fields the engine reads; it's an **open bag**, so forward-compat
|
|
402
|
+
* fields (e.g. `greeting` / `language` once the engine acks them) pass straight
|
|
403
|
+
* through. The SDK supplies the `{"type":"configure"}` envelope for you — which
|
|
404
|
+
* is the whole point: a hand-rolled `{"event":"configure"}` is acked but
|
|
405
|
+
* silently dropped, leaving the agent with no brain and zero turns.
|
|
406
|
+
*/
|
|
407
|
+
export interface OmniConfigure {
|
|
408
|
+
/** Voice to speak with (stock / clone / designed id). */
|
|
409
|
+
voice_id?: string;
|
|
410
|
+
/** System prompt / role + instructions for the agent. */
|
|
411
|
+
persona?: string;
|
|
412
|
+
/** Customer-hosted URL the engine calls per turn for grounding. */
|
|
413
|
+
kb_endpoint?: string;
|
|
414
|
+
/** Bearer the engine presents to `kb_endpoint`. */
|
|
415
|
+
kb_token?: string;
|
|
416
|
+
/** Forward-compatible: any other key the engine honors. */
|
|
417
|
+
[k: string]: unknown;
|
|
418
|
+
}
|
|
419
|
+
export interface OmniConnectOptions {
|
|
420
|
+
/** Optional opaque per-session tag echoed to your `kb_endpoint`. Omni is zero-state. */
|
|
421
|
+
sessionLabel?: string;
|
|
422
|
+
/**
|
|
423
|
+
* Ephemeral browser token from `omni.createSession`. When set it is used as
|
|
424
|
+
* the WS subprotocol (`pyai-key.<token>`) instead of the client's secret key,
|
|
425
|
+
* so a page never holds a secret key.
|
|
426
|
+
*/
|
|
427
|
+
token?: string;
|
|
428
|
+
/** Connect-URL sample rate: 24000 browser, 16000 wideband telephony, 8000 G.711/Twilio. */
|
|
429
|
+
rate?: 24000 | 16000 | 8000;
|
|
430
|
+
/** Connect-URL audio format. Default `pcm16`. */
|
|
431
|
+
format?: "pcm16";
|
|
432
|
+
/**
|
|
433
|
+
* Agent config sent as `{"type":"configure",...}` the instant the socket
|
|
434
|
+
* opens. Omit to send it yourself later via {@link OmniConnection.configure}.
|
|
435
|
+
*/
|
|
436
|
+
configure?: OmniConfigure;
|
|
437
|
+
/** Extra query params merged onto the connect URL. */
|
|
438
|
+
query?: Record<string, string>;
|
|
439
|
+
/** Fired once the socket opens (after the optional auto-configure). */
|
|
440
|
+
onOpen?: () => void;
|
|
441
|
+
/** Fired for each binary agent-audio chunk — play it out as it arrives. */
|
|
442
|
+
onAudio?: (chunk: OmniAudioChunk) => void;
|
|
443
|
+
/** Fired on the `hello` handshake frame. */
|
|
444
|
+
onHello?: (frame: OmniServerFrame) => void;
|
|
445
|
+
/** Fired on the `configured` ack. */
|
|
446
|
+
onConfigured?: (frame: OmniServerFrame) => void;
|
|
447
|
+
/** Fired on `session_started`. */
|
|
448
|
+
onSessionStarted?: (frame: OmniServerFrame) => void;
|
|
449
|
+
/** Fired on `turn` boundaries. */
|
|
450
|
+
onTurn?: (frame: OmniServerFrame) => void;
|
|
451
|
+
/** Fired on `transcript` text frames. */
|
|
452
|
+
onTranscript?: (frame: OmniServerFrame) => void;
|
|
453
|
+
/** Fired on `barge_in` / `flush` (user interrupted). */
|
|
454
|
+
onBargeIn?: (frame: OmniServerFrame) => void;
|
|
455
|
+
/** Fired on `session_end`. */
|
|
456
|
+
onSessionEnd?: (frame: OmniServerFrame) => void;
|
|
457
|
+
/** Fired on EVERY JSON frame (including unknown/forward-compat ones). */
|
|
458
|
+
onEvent?: (frame: OmniServerFrame) => void;
|
|
459
|
+
/** Fired on an `error` frame or a transport-level error. */
|
|
460
|
+
onError?: (err: OmniServerFrame | Error) => void;
|
|
461
|
+
/** Fired when the socket closes (code per {@link WSCloseCode}). */
|
|
462
|
+
onClose?: (code: number, reason: string) => void;
|
|
463
|
+
/** Injectable WebSocket constructor (defaults to the global). */
|
|
464
|
+
webSocket?: WebSocketCtor;
|
|
465
|
+
}
|
|
466
|
+
/**
|
|
467
|
+
* A live Omni agentic-voice session over `/v1/omni`. Hides the wire protocol's
|
|
468
|
+
* frame-key asymmetry: it sends control frames keyed on `type` (`configure`,
|
|
469
|
+
* `dtmf`) and parses server frames keyed on `event`, so you cannot trip the #1
|
|
470
|
+
* Omni integration bug (mirroring the server's `event` key on outbound, which
|
|
471
|
+
* is silently dropped). Construct via `pyai.omni.connect()`.
|
|
472
|
+
*
|
|
473
|
+
* @example
|
|
474
|
+
* const omni = pyai.omni.connect({
|
|
475
|
+
* rate: 16000,
|
|
476
|
+
* configure: { voice_id: "stock_emma_en_gb", persona: "You are a receptionist." },
|
|
477
|
+
* onAudio: (chunk) => speaker.write(chunk),
|
|
478
|
+
* onTranscript: (f) => console.log(f.text),
|
|
479
|
+
* });
|
|
480
|
+
* omni.sendAudio(pcm16Chunk); // stream caller audio continuously
|
|
481
|
+
*/
|
|
482
|
+
export declare class OmniConnection {
|
|
483
|
+
private readonly ws;
|
|
484
|
+
private readonly opts;
|
|
485
|
+
private closed;
|
|
486
|
+
constructor(url: string, subprotocol: string, opts: OmniConnectOptions);
|
|
487
|
+
private handleMessage;
|
|
488
|
+
/**
|
|
489
|
+
* Send (or update) the agent config. Always emitted as
|
|
490
|
+
* `{"type":"configure", ...}` — the correct key. (A hand-rolled
|
|
491
|
+
* `{"event":"configure"}` is acked but silently dropped by the engine.)
|
|
492
|
+
*/
|
|
493
|
+
configure(cfg: OmniConfigure): void;
|
|
494
|
+
/** Stream a chunk of caller audio (PCM16 LE mono at the negotiated rate). */
|
|
495
|
+
sendAudio(chunk: ArrayBufferLike | ArrayBufferView | Blob): void;
|
|
496
|
+
/** Send DTMF digits as a `{"type":"dtmf"}` control frame. */
|
|
497
|
+
sendDtmf(digits: string): void;
|
|
498
|
+
/**
|
|
499
|
+
* Send an arbitrary control frame for forward-compat control types the SDK
|
|
500
|
+
* does not model yet. Reminder: client → server frames are keyed on `type`,
|
|
501
|
+
* never `event`.
|
|
502
|
+
*/
|
|
503
|
+
send(frame: Record<string, unknown>): void;
|
|
504
|
+
/** Close the session. */
|
|
505
|
+
close(code?: number, reason?: string): void;
|
|
506
|
+
/** The underlying socket (escape hatch for advanced use). */
|
|
507
|
+
get socket(): WebSocketLike;
|
|
508
|
+
/** Current WebSocket readyState. */
|
|
509
|
+
get readyState(): number;
|
|
510
|
+
}
|
|
287
511
|
/** Shape of `GET /v1/me`. Fields are best-effort / forward-compatible. */
|
|
288
512
|
export interface MeResponse {
|
|
289
513
|
object?: string;
|
|
@@ -744,6 +968,29 @@ export declare class PyAI {
|
|
|
744
968
|
trigger: (callId: string, input: RecapCallTriggerInput) => Promise<RecapCallSummary>;
|
|
745
969
|
};
|
|
746
970
|
};
|
|
971
|
+
omni: {
|
|
972
|
+
/**
|
|
973
|
+
* Mint an ephemeral, origin-locked Omni session token a browser can use to
|
|
974
|
+
* open ONE realtime session **directly** — the public/private split for
|
|
975
|
+
* realtime. **Call this from your server** with a secret key holding
|
|
976
|
+
* `omni:session`; never ship the secret key to a page. Hand the returned
|
|
977
|
+
* `token` to the browser, which connects with
|
|
978
|
+
* `new WebSocket(session.url, ["pyai-key." + session.token])`. The token
|
|
979
|
+
* expires after `ttlSeconds` (default 60s) and only works from
|
|
980
|
+
* `allowedOrigins`. Scope `omni:session`.
|
|
981
|
+
*/
|
|
982
|
+
createSession: (params: OmniSessionParams) => Promise<OmniSession>;
|
|
983
|
+
/**
|
|
984
|
+
* Open a live Omni agentic-voice session over `/v1/omni`. Returns an
|
|
985
|
+
* {@link OmniConnection} that handles the wire protocol's frame-key
|
|
986
|
+
* asymmetry for you — it sends `type`-keyed control frames (`configure`,
|
|
987
|
+
* `dtmf`) and parses `event`-keyed server frames — so you can't trip the #1
|
|
988
|
+
* Omni integration bug. Omni is zero-state: nothing to create first; the
|
|
989
|
+
* agent's behavior travels in the `configure` frame. Pass `token` (from
|
|
990
|
+
* `createSession`) to connect from a browser without the secret key.
|
|
991
|
+
*/
|
|
992
|
+
connect: (opts?: OmniConnectOptions) => OmniConnection;
|
|
993
|
+
};
|
|
747
994
|
/** Build the realtime WebSocket URL for the chosen product. */
|
|
748
995
|
realtimeURL(opts?: RealtimeOptions): string;
|
|
749
996
|
/** The subprotocol that carries the key on a WS upgrade (browser-safe auth). */
|
package/dist/index.js
CHANGED
|
@@ -45,6 +45,8 @@ export const HearFrameType = {
|
|
|
45
45
|
SpeechFinal: "speech_final",
|
|
46
46
|
/** Corrected, full-context transcript following `speech_final`. */
|
|
47
47
|
Final: "final",
|
|
48
|
+
/** Final billed-usage summary, emitted just before a graceful close. */
|
|
49
|
+
Usage: "usage",
|
|
48
50
|
/** Server-side fault frame. */
|
|
49
51
|
Error: "error",
|
|
50
52
|
};
|
|
@@ -99,7 +101,14 @@ export class HearStream {
|
|
|
99
101
|
this.ws.onopen = () => {
|
|
100
102
|
if (opts.grounding) {
|
|
101
103
|
try {
|
|
102
|
-
|
|
104
|
+
const cfg = { type: "config", grounding: true };
|
|
105
|
+
if (opts.groundingK != null)
|
|
106
|
+
cfg.grounding_k = opts.groundingK;
|
|
107
|
+
if (opts.groundingMinScore != null)
|
|
108
|
+
cfg.grounding_min_score = opts.groundingMinScore;
|
|
109
|
+
if (opts.groundingTimeoutMs != null)
|
|
110
|
+
cfg.grounding_timeout_ms = opts.groundingTimeoutMs;
|
|
111
|
+
this.ws.send(JSON.stringify(cfg));
|
|
103
112
|
}
|
|
104
113
|
catch {
|
|
105
114
|
/* surfaced via onerror */
|
|
@@ -135,6 +144,9 @@ export class HearStream {
|
|
|
135
144
|
case HearFrameType.Final:
|
|
136
145
|
this.opts.onFinal?.(frame);
|
|
137
146
|
break;
|
|
147
|
+
case HearFrameType.Usage:
|
|
148
|
+
this.opts.onUsage?.(frame);
|
|
149
|
+
break;
|
|
138
150
|
case HearFrameType.Error:
|
|
139
151
|
this.opts.onError?.(frame);
|
|
140
152
|
break;
|
|
@@ -165,6 +177,164 @@ export class HearStream {
|
|
|
165
177
|
return this.ws.readyState;
|
|
166
178
|
}
|
|
167
179
|
}
|
|
180
|
+
/* ------------------------------------------------------------------------- *
|
|
181
|
+
* Omni realtime (agentic voice) — typed client over the /v1/omni WebSocket
|
|
182
|
+
* ------------------------------------------------------------------------- */
|
|
183
|
+
/**
|
|
184
|
+
* Event names on Omni **server → client** frames. ⚠️ Note the asymmetry:
|
|
185
|
+
* inbound frames are keyed on `event`, but your **outbound** control frames
|
|
186
|
+
* (`configure`, `dtmf`, …) are keyed on `type`. {@link OmniConnection} handles
|
|
187
|
+
* both sides for you; this map is for matching frames in `onEvent`.
|
|
188
|
+
*/
|
|
189
|
+
export const OmniEvent = {
|
|
190
|
+
/** Handshake; advertises protocol version + audio formats. */
|
|
191
|
+
Hello: "hello",
|
|
192
|
+
/** Ack for your `configure` frame (echoes the resolved `voice_id`). */
|
|
193
|
+
Configured: "configured",
|
|
194
|
+
/** Session is live; includes the resolved agent + audio caps. */
|
|
195
|
+
SessionStarted: "session_started",
|
|
196
|
+
/** Turn boundary (user/assistant speaking). */
|
|
197
|
+
Turn: "turn",
|
|
198
|
+
/** Incremental/final transcript text. */
|
|
199
|
+
Transcript: "transcript",
|
|
200
|
+
/** User interrupted; assistant audio is being cut. */
|
|
201
|
+
BargeIn: "barge_in",
|
|
202
|
+
/** Alias for barge-in on some engine builds. */
|
|
203
|
+
Flush: "flush",
|
|
204
|
+
/** Session is closing; see close code. */
|
|
205
|
+
SessionEnd: "session_end",
|
|
206
|
+
/** Server fault frame. */
|
|
207
|
+
Error: "error",
|
|
208
|
+
};
|
|
209
|
+
/**
|
|
210
|
+
* A live Omni agentic-voice session over `/v1/omni`. Hides the wire protocol's
|
|
211
|
+
* frame-key asymmetry: it sends control frames keyed on `type` (`configure`,
|
|
212
|
+
* `dtmf`) and parses server frames keyed on `event`, so you cannot trip the #1
|
|
213
|
+
* Omni integration bug (mirroring the server's `event` key on outbound, which
|
|
214
|
+
* is silently dropped). Construct via `pyai.omni.connect()`.
|
|
215
|
+
*
|
|
216
|
+
* @example
|
|
217
|
+
* const omni = pyai.omni.connect({
|
|
218
|
+
* rate: 16000,
|
|
219
|
+
* configure: { voice_id: "stock_emma_en_gb", persona: "You are a receptionist." },
|
|
220
|
+
* onAudio: (chunk) => speaker.write(chunk),
|
|
221
|
+
* onTranscript: (f) => console.log(f.text),
|
|
222
|
+
* });
|
|
223
|
+
* omni.sendAudio(pcm16Chunk); // stream caller audio continuously
|
|
224
|
+
*/
|
|
225
|
+
export class OmniConnection {
|
|
226
|
+
ws;
|
|
227
|
+
opts;
|
|
228
|
+
closed = false;
|
|
229
|
+
constructor(url, subprotocol, opts) {
|
|
230
|
+
this.opts = opts;
|
|
231
|
+
const WS = opts.webSocket ?? globalThis.WebSocket;
|
|
232
|
+
if (!WS) {
|
|
233
|
+
throw new Error("No global WebSocket available; pass options.webSocket (e.g. the `ws` package) to omni.connect()");
|
|
234
|
+
}
|
|
235
|
+
this.ws = new WS(url, [subprotocol]);
|
|
236
|
+
this.ws.onopen = () => {
|
|
237
|
+
if (opts.configure) {
|
|
238
|
+
try {
|
|
239
|
+
this.configure(opts.configure);
|
|
240
|
+
}
|
|
241
|
+
catch {
|
|
242
|
+
/* surfaced via onerror */
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
opts.onOpen?.();
|
|
246
|
+
};
|
|
247
|
+
this.ws.onmessage = (ev) => this.handleMessage(ev.data);
|
|
248
|
+
this.ws.onerror = (ev) => opts.onError?.(ev instanceof Error ? ev : new Error("WebSocket error"));
|
|
249
|
+
this.ws.onclose = (ev) => {
|
|
250
|
+
this.closed = true;
|
|
251
|
+
opts.onClose?.(ev.code, ev.reason);
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
handleMessage(data) {
|
|
255
|
+
// Binary = agent audio (play it out). JSON text = an `event`-keyed frame.
|
|
256
|
+
if (typeof data !== "string") {
|
|
257
|
+
this.opts.onAudio?.(data);
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
let frame;
|
|
261
|
+
try {
|
|
262
|
+
frame = JSON.parse(data);
|
|
263
|
+
}
|
|
264
|
+
catch {
|
|
265
|
+
this.opts.onError?.(new Error(`Unparseable Omni frame: ${data.slice(0, 120)}`));
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
this.opts.onEvent?.(frame);
|
|
269
|
+
switch (frame.event) {
|
|
270
|
+
case OmniEvent.Hello:
|
|
271
|
+
this.opts.onHello?.(frame);
|
|
272
|
+
break;
|
|
273
|
+
case OmniEvent.Configured:
|
|
274
|
+
this.opts.onConfigured?.(frame);
|
|
275
|
+
break;
|
|
276
|
+
case OmniEvent.SessionStarted:
|
|
277
|
+
this.opts.onSessionStarted?.(frame);
|
|
278
|
+
break;
|
|
279
|
+
case OmniEvent.Turn:
|
|
280
|
+
this.opts.onTurn?.(frame);
|
|
281
|
+
break;
|
|
282
|
+
case OmniEvent.Transcript:
|
|
283
|
+
this.opts.onTranscript?.(frame);
|
|
284
|
+
break;
|
|
285
|
+
case OmniEvent.BargeIn:
|
|
286
|
+
case OmniEvent.Flush:
|
|
287
|
+
this.opts.onBargeIn?.(frame);
|
|
288
|
+
break;
|
|
289
|
+
case OmniEvent.SessionEnd:
|
|
290
|
+
this.opts.onSessionEnd?.(frame);
|
|
291
|
+
break;
|
|
292
|
+
case OmniEvent.Error:
|
|
293
|
+
this.opts.onError?.(frame);
|
|
294
|
+
break;
|
|
295
|
+
default:
|
|
296
|
+
// Unknown/forward-compatible frame — already delivered via onEvent.
|
|
297
|
+
break;
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Send (or update) the agent config. Always emitted as
|
|
302
|
+
* `{"type":"configure", ...}` — the correct key. (A hand-rolled
|
|
303
|
+
* `{"event":"configure"}` is acked but silently dropped by the engine.)
|
|
304
|
+
*/
|
|
305
|
+
configure(cfg) {
|
|
306
|
+
this.ws.send(JSON.stringify({ type: "configure", ...cfg }));
|
|
307
|
+
}
|
|
308
|
+
/** Stream a chunk of caller audio (PCM16 LE mono at the negotiated rate). */
|
|
309
|
+
sendAudio(chunk) {
|
|
310
|
+
this.ws.send(chunk);
|
|
311
|
+
}
|
|
312
|
+
/** Send DTMF digits as a `{"type":"dtmf"}` control frame. */
|
|
313
|
+
sendDtmf(digits) {
|
|
314
|
+
this.ws.send(JSON.stringify({ type: "dtmf", digits }));
|
|
315
|
+
}
|
|
316
|
+
/**
|
|
317
|
+
* Send an arbitrary control frame for forward-compat control types the SDK
|
|
318
|
+
* does not model yet. Reminder: client → server frames are keyed on `type`,
|
|
319
|
+
* never `event`.
|
|
320
|
+
*/
|
|
321
|
+
send(frame) {
|
|
322
|
+
this.ws.send(JSON.stringify(frame));
|
|
323
|
+
}
|
|
324
|
+
/** Close the session. */
|
|
325
|
+
close(code = WSCloseCode.Normal, reason = "") {
|
|
326
|
+
if (!this.closed)
|
|
327
|
+
this.ws.close(code, reason);
|
|
328
|
+
}
|
|
329
|
+
/** The underlying socket (escape hatch for advanced use). */
|
|
330
|
+
get socket() {
|
|
331
|
+
return this.ws;
|
|
332
|
+
}
|
|
333
|
+
/** Current WebSocket readyState. */
|
|
334
|
+
get readyState() {
|
|
335
|
+
return this.ws.readyState;
|
|
336
|
+
}
|
|
337
|
+
}
|
|
168
338
|
const RETRYABLE = new Set([429, 500, 502, 503, 504]);
|
|
169
339
|
export class PyAI {
|
|
170
340
|
apiKey;
|
|
@@ -510,17 +680,57 @@ export class PyAI {
|
|
|
510
680
|
trigger: (callId, input) => this.postJson(`/v1/recap/calls/${encodeURIComponent(callId)}`, input),
|
|
511
681
|
},
|
|
512
682
|
};
|
|
683
|
+
// --- omni (agentic voice) ----------------------------------------------
|
|
684
|
+
omni = {
|
|
685
|
+
/**
|
|
686
|
+
* Mint an ephemeral, origin-locked Omni session token a browser can use to
|
|
687
|
+
* open ONE realtime session **directly** — the public/private split for
|
|
688
|
+
* realtime. **Call this from your server** with a secret key holding
|
|
689
|
+
* `omni:session`; never ship the secret key to a page. Hand the returned
|
|
690
|
+
* `token` to the browser, which connects with
|
|
691
|
+
* `new WebSocket(session.url, ["pyai-key." + session.token])`. The token
|
|
692
|
+
* expires after `ttlSeconds` (default 60s) and only works from
|
|
693
|
+
* `allowedOrigins`. Scope `omni:session`.
|
|
694
|
+
*/
|
|
695
|
+
createSession: (params) => this.postJson("/v1/omni/sessions", {
|
|
696
|
+
allowed_origins: params.allowedOrigins,
|
|
697
|
+
...(params.ttlSeconds !== undefined ? { ttl_seconds: params.ttlSeconds } : {}),
|
|
698
|
+
...(params.sessionLabel !== undefined ? { session_label: params.sessionLabel } : {}),
|
|
699
|
+
}),
|
|
700
|
+
/**
|
|
701
|
+
* Open a live Omni agentic-voice session over `/v1/omni`. Returns an
|
|
702
|
+
* {@link OmniConnection} that handles the wire protocol's frame-key
|
|
703
|
+
* asymmetry for you — it sends `type`-keyed control frames (`configure`,
|
|
704
|
+
* `dtmf`) and parses `event`-keyed server frames — so you can't trip the #1
|
|
705
|
+
* Omni integration bug. Omni is zero-state: nothing to create first; the
|
|
706
|
+
* agent's behavior travels in the `configure` frame. Pass `token` (from
|
|
707
|
+
* `createSession`) to connect from a browser without the secret key.
|
|
708
|
+
*/
|
|
709
|
+
connect: (opts = {}) => {
|
|
710
|
+
const query = { ...(opts.query ?? {}) };
|
|
711
|
+
if (opts.format)
|
|
712
|
+
query.format = opts.format;
|
|
713
|
+
if (opts.rate)
|
|
714
|
+
query.rate = String(opts.rate);
|
|
715
|
+
const url = this.realtimeURL({ product: "omni", sessionLabel: opts.sessionLabel, query });
|
|
716
|
+
const sub = opts.token ? `pyai-key.${opts.token}` : this.realtimeSubprotocol();
|
|
717
|
+
return new OmniConnection(url, sub, opts);
|
|
718
|
+
},
|
|
719
|
+
};
|
|
513
720
|
// --- realtime (WebSocket) ----------------------------------------------
|
|
514
721
|
/** Build the realtime WebSocket URL for the chosen product. */
|
|
515
722
|
realtimeURL(opts = {}) {
|
|
516
723
|
const wsBase = this.baseURL.replace(/^http/, "ws");
|
|
517
724
|
const q = new URLSearchParams(opts.query ?? {});
|
|
518
725
|
if ((opts.product ?? "omni") === "omni") {
|
|
519
|
-
// Omni's native realtime surface is /v1/omni.
|
|
520
|
-
//
|
|
521
|
-
// connect URL, so default to
|
|
522
|
-
|
|
523
|
-
|
|
726
|
+
// Omni's native realtime surface is /v1/omni. The session is authorized by
|
|
727
|
+
// the key's org (zero-state) — sessionLabel is an optional opaque tag.
|
|
728
|
+
// format/rate are load-bearing on the connect URL, so default to
|
|
729
|
+
// browser-grade PCM16/24kHz.
|
|
730
|
+
if (opts.sessionLabel)
|
|
731
|
+
q.set("session_label", opts.sessionLabel);
|
|
732
|
+
else if (opts.agentId)
|
|
733
|
+
q.set("agent_id", opts.agentId); // deprecated alias
|
|
524
734
|
if (!q.has("format"))
|
|
525
735
|
q.set("format", "pcm16");
|
|
526
736
|
if (!q.has("rate"))
|
|
@@ -549,6 +759,10 @@ export class PyAI {
|
|
|
549
759
|
q.set("encoding", opts.encoding);
|
|
550
760
|
if (opts.interimResults !== undefined)
|
|
551
761
|
q.set("interim_results", String(opts.interimResults));
|
|
762
|
+
if (opts.numerals !== undefined)
|
|
763
|
+
q.set("numerals", String(opts.numerals));
|
|
764
|
+
if (opts.endpointingMs !== undefined)
|
|
765
|
+
q.set("endpointing_ms", String(opts.endpointingMs));
|
|
552
766
|
const qs = q.toString();
|
|
553
767
|
return `${wsBase}/v1/audio/transcriptions/stream${qs ? `?${qs}` : ""}`;
|
|
554
768
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pyai/sdk",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "Official TypeScript/JavaScript SDK for PyAI — speech-to-text (Hear), text-to-speech (Speak), realtime voice agents (Omni), and call compliance (Trace).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
},
|
|
23
23
|
"homepage": "https://pyai.com",
|
|
24
24
|
"bugs": "https://github.com/atomsai/pyai-platform-backend/issues",
|
|
25
|
+
"author": "PyAI",
|
|
25
26
|
"scripts": {
|
|
26
27
|
"build": "tsc -p tsconfig.build.json",
|
|
27
28
|
"test": "node --test test/**/*.test.ts",
|
package/src/index.ts
CHANGED
|
@@ -139,14 +139,55 @@ export interface CreateJobParams {
|
|
|
139
139
|
}
|
|
140
140
|
|
|
141
141
|
export interface RealtimeOptions {
|
|
142
|
-
/** "omni" (agentic
|
|
142
|
+
/** "omni" (agentic voice) or "flow" (legacy voice duplex). Default "omni". */
|
|
143
143
|
product?: "omni" | "flow";
|
|
144
|
-
/**
|
|
144
|
+
/**
|
|
145
|
+
* Optional opaque tag echoed to your `kb_endpoint` and recorded on the call.
|
|
146
|
+
* Omni is zero-state: the session is authorized by the key's org, so there is
|
|
147
|
+
* nothing to create first and this tag is never required.
|
|
148
|
+
*/
|
|
149
|
+
sessionLabel?: string;
|
|
150
|
+
/**
|
|
151
|
+
* @deprecated Use {@link sessionLabel}. Kept for back-compat — emitted as the
|
|
152
|
+
* `agent_id` query alias, which the gateway still accepts. Ignored if
|
|
153
|
+
* `sessionLabel` is set.
|
|
154
|
+
*/
|
|
145
155
|
agentId?: string;
|
|
146
156
|
/** Extra query params (e.g. format, rate). */
|
|
147
157
|
query?: Record<string, string>;
|
|
148
158
|
}
|
|
149
159
|
|
|
160
|
+
/** Parameters for minting an ephemeral browser Omni session token (server-side). */
|
|
161
|
+
export interface OmniSessionParams {
|
|
162
|
+
/**
|
|
163
|
+
* Browser origins (scheme://host[:port]) the minted token may connect from.
|
|
164
|
+
* Required and non-empty — a browser token must be origin-locked. `*` is not
|
|
165
|
+
* allowed.
|
|
166
|
+
*/
|
|
167
|
+
allowedOrigins: string[];
|
|
168
|
+
/** Token lifetime in seconds. Default 60 server-side; max 600. Keep it short. */
|
|
169
|
+
ttlSeconds?: number;
|
|
170
|
+
/** Optional opaque tag echoed back to your `kb_endpoint` and recorded on the call. */
|
|
171
|
+
sessionLabel?: string;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* The result of {@link PyAI.omni}.createSession — a short-lived credential safe
|
|
176
|
+
* to hand to the browser. Use {@link OmniSession.token} as the WebSocket
|
|
177
|
+
* subprotocol `pyai-key.<token>` against {@link OmniSession.url}.
|
|
178
|
+
*/
|
|
179
|
+
export interface OmniSession {
|
|
180
|
+
object?: "omni.session";
|
|
181
|
+
/** The ephemeral session token. Short-lived + origin-locked; browser-safe. */
|
|
182
|
+
token: string;
|
|
183
|
+
/** Token expiry, Unix epoch milliseconds. */
|
|
184
|
+
expires_at: number;
|
|
185
|
+
/** The Omni realtime WebSocket URL to connect to. */
|
|
186
|
+
url: string;
|
|
187
|
+
/** Echoed back when supplied on the request. */
|
|
188
|
+
session_label?: string;
|
|
189
|
+
}
|
|
190
|
+
|
|
150
191
|
/* ------------------------------------------------------------------------- *
|
|
151
192
|
* Stable enums — mirror the server so callers branch on named constants, not
|
|
152
193
|
* magic strings, and a contract change surfaces in one place. These are plain
|
|
@@ -164,6 +205,8 @@ export const HearFrameType = {
|
|
|
164
205
|
SpeechFinal: "speech_final",
|
|
165
206
|
/** Corrected, full-context transcript following `speech_final`. */
|
|
166
207
|
Final: "final",
|
|
208
|
+
/** Final billed-usage summary, emitted just before a graceful close. */
|
|
209
|
+
Usage: "usage",
|
|
167
210
|
/** Server-side fault frame. */
|
|
168
211
|
Error: "error",
|
|
169
212
|
} as const;
|
|
@@ -239,6 +282,22 @@ export interface HearFinalFrame {
|
|
|
239
282
|
grounding?: HearGroundingPassage[];
|
|
240
283
|
}
|
|
241
284
|
|
|
285
|
+
/** Final billed-usage summary (`usage`), emitted just before a graceful close
|
|
286
|
+
* so realtime spend can be reconciled in-band (a realtime WS carries no
|
|
287
|
+
* `x-pyai-units` response header). Best-effort: absent if there was no billable
|
|
288
|
+
* audio or the close was abnormal. */
|
|
289
|
+
export interface HearUsageFrame {
|
|
290
|
+
type: "usage";
|
|
291
|
+
/** `hear` for plain streaming, `cue` when grounding was enabled. */
|
|
292
|
+
product: "hear" | "cue";
|
|
293
|
+
/** The billed meter (`hear.requests` or `cue.minutes`). */
|
|
294
|
+
meter: string;
|
|
295
|
+
/** Summed active-speech audio billed for the session, in seconds. */
|
|
296
|
+
audio_seconds: number;
|
|
297
|
+
/** The same quantity in minutes. */
|
|
298
|
+
minutes: number;
|
|
299
|
+
}
|
|
300
|
+
|
|
242
301
|
/** Server fault frame (`error`). */
|
|
243
302
|
export interface HearErrorFrame {
|
|
244
303
|
type: "error";
|
|
@@ -246,7 +305,7 @@ export interface HearErrorFrame {
|
|
|
246
305
|
message: string;
|
|
247
306
|
}
|
|
248
307
|
|
|
249
|
-
export type HearFrame = HearPartialFrame | HearFinalFrame | HearErrorFrame;
|
|
308
|
+
export type HearFrame = HearPartialFrame | HearFinalFrame | HearUsageFrame | HearErrorFrame;
|
|
250
309
|
|
|
251
310
|
/**
|
|
252
311
|
* Minimal structural WebSocket — matches both the browser/Node global
|
|
@@ -275,12 +334,32 @@ export interface HearStreamOptions {
|
|
|
275
334
|
encoding?: "pcm16" | "opus";
|
|
276
335
|
/** Emit eager partial hypotheses. Default true server-side. */
|
|
277
336
|
interimResults?: boolean;
|
|
337
|
+
/**
|
|
338
|
+
* Format spoken numbers as digits in the transcript (e.g. "one two three" →
|
|
339
|
+
* "123"). Useful for voice agents that read back phone numbers, codes, and
|
|
340
|
+
* amounts. Default false (spoken form). Forwards `?numerals=true` on the URL.
|
|
341
|
+
*/
|
|
342
|
+
numerals?: boolean;
|
|
343
|
+
/**
|
|
344
|
+
* Turn-segmentation tuning: trailing-pause (ms, 50–2000) that ends an
|
|
345
|
+
* utterance. Forwards `?endpointing_ms=` — clamped + honored once the engine
|
|
346
|
+
* supports it; a no-op when omitted. Drive end-of-turn yourself with
|
|
347
|
+
* {@link HearStream.commit} for full control today.
|
|
348
|
+
*/
|
|
349
|
+
endpointingMs?: number;
|
|
278
350
|
/**
|
|
279
351
|
* Enable Cue knowledge-base grounding: sends `{type:"config",grounding:true}`
|
|
280
352
|
* on open, after which `speech_final`/`final` frames carry a `grounding`
|
|
281
353
|
* array. Bills a single `cue.minutes` line instead of the Hear rate.
|
|
282
354
|
*/
|
|
283
355
|
grounding?: boolean;
|
|
356
|
+
/** Cue: number of KB passages to retrieve per turn (1–20, default 3). */
|
|
357
|
+
groundingK?: number;
|
|
358
|
+
/** Cue: drop passages scoring below this (0–1, default 0 = keep all). */
|
|
359
|
+
groundingMinScore?: number;
|
|
360
|
+
/** Cue: max ms to wait for retrieval at the final before failing open to
|
|
361
|
+
* `grounding: []` (50–2000, default 450). */
|
|
362
|
+
groundingTimeoutMs?: number;
|
|
284
363
|
/** Extra query params merged onto the connect URL. */
|
|
285
364
|
query?: Record<string, string>;
|
|
286
365
|
/** Fired once the socket opens (after the optional grounding config). */
|
|
@@ -289,6 +368,8 @@ export interface HearStreamOptions {
|
|
|
289
368
|
onPartial?: (frame: HearPartialFrame) => void;
|
|
290
369
|
/** Fired on `speech_final` / `final`. */
|
|
291
370
|
onFinal?: (frame: HearFinalFrame) => void;
|
|
371
|
+
/** Fired on the final `usage` summary (in-band realtime reconciliation). */
|
|
372
|
+
onUsage?: (frame: HearUsageFrame) => void;
|
|
292
373
|
/** Fired on an `error` frame or a transport-level error. */
|
|
293
374
|
onError?: (err: HearErrorFrame | Error) => void;
|
|
294
375
|
/** Fired when the socket closes (code per `WSCloseCode`). */
|
|
@@ -320,7 +401,11 @@ export class HearStream {
|
|
|
320
401
|
this.ws.onopen = () => {
|
|
321
402
|
if (opts.grounding) {
|
|
322
403
|
try {
|
|
323
|
-
|
|
404
|
+
const cfg: Record<string, unknown> = { type: "config", grounding: true };
|
|
405
|
+
if (opts.groundingK != null) cfg.grounding_k = opts.groundingK;
|
|
406
|
+
if (opts.groundingMinScore != null) cfg.grounding_min_score = opts.groundingMinScore;
|
|
407
|
+
if (opts.groundingTimeoutMs != null) cfg.grounding_timeout_ms = opts.groundingTimeoutMs;
|
|
408
|
+
this.ws.send(JSON.stringify(cfg));
|
|
324
409
|
} catch {
|
|
325
410
|
/* surfaced via onerror */
|
|
326
411
|
}
|
|
@@ -354,6 +439,9 @@ export class HearStream {
|
|
|
354
439
|
case HearFrameType.Final:
|
|
355
440
|
this.opts.onFinal?.(frame);
|
|
356
441
|
break;
|
|
442
|
+
case HearFrameType.Usage:
|
|
443
|
+
this.opts.onUsage?.(frame);
|
|
444
|
+
break;
|
|
357
445
|
case HearFrameType.Error:
|
|
358
446
|
this.opts.onError?.(frame);
|
|
359
447
|
break;
|
|
@@ -389,6 +477,254 @@ export class HearStream {
|
|
|
389
477
|
}
|
|
390
478
|
}
|
|
391
479
|
|
|
480
|
+
/* ------------------------------------------------------------------------- *
|
|
481
|
+
* Omni realtime (agentic voice) — typed client over the /v1/omni WebSocket
|
|
482
|
+
* ------------------------------------------------------------------------- */
|
|
483
|
+
|
|
484
|
+
/**
|
|
485
|
+
* Event names on Omni **server → client** frames. ⚠️ Note the asymmetry:
|
|
486
|
+
* inbound frames are keyed on `event`, but your **outbound** control frames
|
|
487
|
+
* (`configure`, `dtmf`, …) are keyed on `type`. {@link OmniConnection} handles
|
|
488
|
+
* both sides for you; this map is for matching frames in `onEvent`.
|
|
489
|
+
*/
|
|
490
|
+
export const OmniEvent = {
|
|
491
|
+
/** Handshake; advertises protocol version + audio formats. */
|
|
492
|
+
Hello: "hello",
|
|
493
|
+
/** Ack for your `configure` frame (echoes the resolved `voice_id`). */
|
|
494
|
+
Configured: "configured",
|
|
495
|
+
/** Session is live; includes the resolved agent + audio caps. */
|
|
496
|
+
SessionStarted: "session_started",
|
|
497
|
+
/** Turn boundary (user/assistant speaking). */
|
|
498
|
+
Turn: "turn",
|
|
499
|
+
/** Incremental/final transcript text. */
|
|
500
|
+
Transcript: "transcript",
|
|
501
|
+
/** User interrupted; assistant audio is being cut. */
|
|
502
|
+
BargeIn: "barge_in",
|
|
503
|
+
/** Alias for barge-in on some engine builds. */
|
|
504
|
+
Flush: "flush",
|
|
505
|
+
/** Session is closing; see close code. */
|
|
506
|
+
SessionEnd: "session_end",
|
|
507
|
+
/** Server fault frame. */
|
|
508
|
+
Error: "error",
|
|
509
|
+
} as const;
|
|
510
|
+
export type OmniEvent = (typeof OmniEvent)[keyof typeof OmniEvent];
|
|
511
|
+
|
|
512
|
+
/** A server → client Omni frame. Keyed on `event` (not `type`); open for forward-compat. */
|
|
513
|
+
export interface OmniServerFrame {
|
|
514
|
+
event: string;
|
|
515
|
+
[k: string]: unknown;
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/** A binary agent-audio chunk delivered to {@link OmniConnectOptions.onAudio}. */
|
|
519
|
+
export type OmniAudioChunk = ArrayBuffer | ArrayBufferView | Blob;
|
|
520
|
+
|
|
521
|
+
/**
|
|
522
|
+
* The agent config sent as the `configure` control frame. These are the wire
|
|
523
|
+
* (snake_case) fields the engine reads; it's an **open bag**, so forward-compat
|
|
524
|
+
* fields (e.g. `greeting` / `language` once the engine acks them) pass straight
|
|
525
|
+
* through. The SDK supplies the `{"type":"configure"}` envelope for you — which
|
|
526
|
+
* is the whole point: a hand-rolled `{"event":"configure"}` is acked but
|
|
527
|
+
* silently dropped, leaving the agent with no brain and zero turns.
|
|
528
|
+
*/
|
|
529
|
+
export interface OmniConfigure {
|
|
530
|
+
/** Voice to speak with (stock / clone / designed id). */
|
|
531
|
+
voice_id?: string;
|
|
532
|
+
/** System prompt / role + instructions for the agent. */
|
|
533
|
+
persona?: string;
|
|
534
|
+
/** Customer-hosted URL the engine calls per turn for grounding. */
|
|
535
|
+
kb_endpoint?: string;
|
|
536
|
+
/** Bearer the engine presents to `kb_endpoint`. */
|
|
537
|
+
kb_token?: string;
|
|
538
|
+
/** Forward-compatible: any other key the engine honors. */
|
|
539
|
+
[k: string]: unknown;
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
export interface OmniConnectOptions {
|
|
543
|
+
/** Optional opaque per-session tag echoed to your `kb_endpoint`. Omni is zero-state. */
|
|
544
|
+
sessionLabel?: string;
|
|
545
|
+
/**
|
|
546
|
+
* Ephemeral browser token from `omni.createSession`. When set it is used as
|
|
547
|
+
* the WS subprotocol (`pyai-key.<token>`) instead of the client's secret key,
|
|
548
|
+
* so a page never holds a secret key.
|
|
549
|
+
*/
|
|
550
|
+
token?: string;
|
|
551
|
+
/** Connect-URL sample rate: 24000 browser, 16000 wideband telephony, 8000 G.711/Twilio. */
|
|
552
|
+
rate?: 24000 | 16000 | 8000;
|
|
553
|
+
/** Connect-URL audio format. Default `pcm16`. */
|
|
554
|
+
format?: "pcm16";
|
|
555
|
+
/**
|
|
556
|
+
* Agent config sent as `{"type":"configure",...}` the instant the socket
|
|
557
|
+
* opens. Omit to send it yourself later via {@link OmniConnection.configure}.
|
|
558
|
+
*/
|
|
559
|
+
configure?: OmniConfigure;
|
|
560
|
+
/** Extra query params merged onto the connect URL. */
|
|
561
|
+
query?: Record<string, string>;
|
|
562
|
+
/** Fired once the socket opens (after the optional auto-configure). */
|
|
563
|
+
onOpen?: () => void;
|
|
564
|
+
/** Fired for each binary agent-audio chunk — play it out as it arrives. */
|
|
565
|
+
onAudio?: (chunk: OmniAudioChunk) => void;
|
|
566
|
+
/** Fired on the `hello` handshake frame. */
|
|
567
|
+
onHello?: (frame: OmniServerFrame) => void;
|
|
568
|
+
/** Fired on the `configured` ack. */
|
|
569
|
+
onConfigured?: (frame: OmniServerFrame) => void;
|
|
570
|
+
/** Fired on `session_started`. */
|
|
571
|
+
onSessionStarted?: (frame: OmniServerFrame) => void;
|
|
572
|
+
/** Fired on `turn` boundaries. */
|
|
573
|
+
onTurn?: (frame: OmniServerFrame) => void;
|
|
574
|
+
/** Fired on `transcript` text frames. */
|
|
575
|
+
onTranscript?: (frame: OmniServerFrame) => void;
|
|
576
|
+
/** Fired on `barge_in` / `flush` (user interrupted). */
|
|
577
|
+
onBargeIn?: (frame: OmniServerFrame) => void;
|
|
578
|
+
/** Fired on `session_end`. */
|
|
579
|
+
onSessionEnd?: (frame: OmniServerFrame) => void;
|
|
580
|
+
/** Fired on EVERY JSON frame (including unknown/forward-compat ones). */
|
|
581
|
+
onEvent?: (frame: OmniServerFrame) => void;
|
|
582
|
+
/** Fired on an `error` frame or a transport-level error. */
|
|
583
|
+
onError?: (err: OmniServerFrame | Error) => void;
|
|
584
|
+
/** Fired when the socket closes (code per {@link WSCloseCode}). */
|
|
585
|
+
onClose?: (code: number, reason: string) => void;
|
|
586
|
+
/** Injectable WebSocket constructor (defaults to the global). */
|
|
587
|
+
webSocket?: WebSocketCtor;
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
/**
|
|
591
|
+
* A live Omni agentic-voice session over `/v1/omni`. Hides the wire protocol's
|
|
592
|
+
* frame-key asymmetry: it sends control frames keyed on `type` (`configure`,
|
|
593
|
+
* `dtmf`) and parses server frames keyed on `event`, so you cannot trip the #1
|
|
594
|
+
* Omni integration bug (mirroring the server's `event` key on outbound, which
|
|
595
|
+
* is silently dropped). Construct via `pyai.omni.connect()`.
|
|
596
|
+
*
|
|
597
|
+
* @example
|
|
598
|
+
* const omni = pyai.omni.connect({
|
|
599
|
+
* rate: 16000,
|
|
600
|
+
* configure: { voice_id: "stock_emma_en_gb", persona: "You are a receptionist." },
|
|
601
|
+
* onAudio: (chunk) => speaker.write(chunk),
|
|
602
|
+
* onTranscript: (f) => console.log(f.text),
|
|
603
|
+
* });
|
|
604
|
+
* omni.sendAudio(pcm16Chunk); // stream caller audio continuously
|
|
605
|
+
*/
|
|
606
|
+
export class OmniConnection {
|
|
607
|
+
private readonly ws: WebSocketLike;
|
|
608
|
+
private readonly opts: OmniConnectOptions;
|
|
609
|
+
private closed = false;
|
|
610
|
+
|
|
611
|
+
constructor(url: string, subprotocol: string, opts: OmniConnectOptions) {
|
|
612
|
+
this.opts = opts;
|
|
613
|
+
const WS = opts.webSocket ?? (globalThis as { WebSocket?: WebSocketCtor }).WebSocket;
|
|
614
|
+
if (!WS) {
|
|
615
|
+
throw new Error(
|
|
616
|
+
"No global WebSocket available; pass options.webSocket (e.g. the `ws` package) to omni.connect()",
|
|
617
|
+
);
|
|
618
|
+
}
|
|
619
|
+
this.ws = new WS(url, [subprotocol]);
|
|
620
|
+
this.ws.onopen = () => {
|
|
621
|
+
if (opts.configure) {
|
|
622
|
+
try {
|
|
623
|
+
this.configure(opts.configure);
|
|
624
|
+
} catch {
|
|
625
|
+
/* surfaced via onerror */
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
opts.onOpen?.();
|
|
629
|
+
};
|
|
630
|
+
this.ws.onmessage = (ev) => this.handleMessage(ev.data);
|
|
631
|
+
this.ws.onerror = (ev) => opts.onError?.(ev instanceof Error ? ev : new Error("WebSocket error"));
|
|
632
|
+
this.ws.onclose = (ev) => {
|
|
633
|
+
this.closed = true;
|
|
634
|
+
opts.onClose?.(ev.code, ev.reason);
|
|
635
|
+
};
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
private handleMessage(data: unknown): void {
|
|
639
|
+
// Binary = agent audio (play it out). JSON text = an `event`-keyed frame.
|
|
640
|
+
if (typeof data !== "string") {
|
|
641
|
+
this.opts.onAudio?.(data as OmniAudioChunk);
|
|
642
|
+
return;
|
|
643
|
+
}
|
|
644
|
+
let frame: OmniServerFrame;
|
|
645
|
+
try {
|
|
646
|
+
frame = JSON.parse(data) as OmniServerFrame;
|
|
647
|
+
} catch {
|
|
648
|
+
this.opts.onError?.(new Error(`Unparseable Omni frame: ${data.slice(0, 120)}`));
|
|
649
|
+
return;
|
|
650
|
+
}
|
|
651
|
+
this.opts.onEvent?.(frame);
|
|
652
|
+
switch (frame.event) {
|
|
653
|
+
case OmniEvent.Hello:
|
|
654
|
+
this.opts.onHello?.(frame);
|
|
655
|
+
break;
|
|
656
|
+
case OmniEvent.Configured:
|
|
657
|
+
this.opts.onConfigured?.(frame);
|
|
658
|
+
break;
|
|
659
|
+
case OmniEvent.SessionStarted:
|
|
660
|
+
this.opts.onSessionStarted?.(frame);
|
|
661
|
+
break;
|
|
662
|
+
case OmniEvent.Turn:
|
|
663
|
+
this.opts.onTurn?.(frame);
|
|
664
|
+
break;
|
|
665
|
+
case OmniEvent.Transcript:
|
|
666
|
+
this.opts.onTranscript?.(frame);
|
|
667
|
+
break;
|
|
668
|
+
case OmniEvent.BargeIn:
|
|
669
|
+
case OmniEvent.Flush:
|
|
670
|
+
this.opts.onBargeIn?.(frame);
|
|
671
|
+
break;
|
|
672
|
+
case OmniEvent.SessionEnd:
|
|
673
|
+
this.opts.onSessionEnd?.(frame);
|
|
674
|
+
break;
|
|
675
|
+
case OmniEvent.Error:
|
|
676
|
+
this.opts.onError?.(frame);
|
|
677
|
+
break;
|
|
678
|
+
default:
|
|
679
|
+
// Unknown/forward-compatible frame — already delivered via onEvent.
|
|
680
|
+
break;
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
/**
|
|
685
|
+
* Send (or update) the agent config. Always emitted as
|
|
686
|
+
* `{"type":"configure", ...}` — the correct key. (A hand-rolled
|
|
687
|
+
* `{"event":"configure"}` is acked but silently dropped by the engine.)
|
|
688
|
+
*/
|
|
689
|
+
configure(cfg: OmniConfigure): void {
|
|
690
|
+
this.ws.send(JSON.stringify({ type: "configure", ...cfg }));
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
/** Stream a chunk of caller audio (PCM16 LE mono at the negotiated rate). */
|
|
694
|
+
sendAudio(chunk: ArrayBufferLike | ArrayBufferView | Blob): void {
|
|
695
|
+
this.ws.send(chunk);
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
/** Send DTMF digits as a `{"type":"dtmf"}` control frame. */
|
|
699
|
+
sendDtmf(digits: string): void {
|
|
700
|
+
this.ws.send(JSON.stringify({ type: "dtmf", digits }));
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
/**
|
|
704
|
+
* Send an arbitrary control frame for forward-compat control types the SDK
|
|
705
|
+
* does not model yet. Reminder: client → server frames are keyed on `type`,
|
|
706
|
+
* never `event`.
|
|
707
|
+
*/
|
|
708
|
+
send(frame: Record<string, unknown>): void {
|
|
709
|
+
this.ws.send(JSON.stringify(frame));
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
/** Close the session. */
|
|
713
|
+
close(code: number = WSCloseCode.Normal, reason = ""): void {
|
|
714
|
+
if (!this.closed) this.ws.close(code, reason);
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
/** The underlying socket (escape hatch for advanced use). */
|
|
718
|
+
get socket(): WebSocketLike {
|
|
719
|
+
return this.ws;
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
/** Current WebSocket readyState. */
|
|
723
|
+
get readyState(): number {
|
|
724
|
+
return this.ws.readyState;
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
|
|
392
728
|
/* ------------------------------------------------------------------------- *
|
|
393
729
|
* Key introspection — GET /v1/me
|
|
394
730
|
* ------------------------------------------------------------------------- */
|
|
@@ -1064,6 +1400,45 @@ export class PyAI {
|
|
|
1064
1400
|
},
|
|
1065
1401
|
};
|
|
1066
1402
|
|
|
1403
|
+
// --- omni (agentic voice) ----------------------------------------------
|
|
1404
|
+
|
|
1405
|
+
omni = {
|
|
1406
|
+
/**
|
|
1407
|
+
* Mint an ephemeral, origin-locked Omni session token a browser can use to
|
|
1408
|
+
* open ONE realtime session **directly** — the public/private split for
|
|
1409
|
+
* realtime. **Call this from your server** with a secret key holding
|
|
1410
|
+
* `omni:session`; never ship the secret key to a page. Hand the returned
|
|
1411
|
+
* `token` to the browser, which connects with
|
|
1412
|
+
* `new WebSocket(session.url, ["pyai-key." + session.token])`. The token
|
|
1413
|
+
* expires after `ttlSeconds` (default 60s) and only works from
|
|
1414
|
+
* `allowedOrigins`. Scope `omni:session`.
|
|
1415
|
+
*/
|
|
1416
|
+
createSession: (params: OmniSessionParams): Promise<OmniSession> =>
|
|
1417
|
+
this.postJson("/v1/omni/sessions", {
|
|
1418
|
+
allowed_origins: params.allowedOrigins,
|
|
1419
|
+
...(params.ttlSeconds !== undefined ? { ttl_seconds: params.ttlSeconds } : {}),
|
|
1420
|
+
...(params.sessionLabel !== undefined ? { session_label: params.sessionLabel } : {}),
|
|
1421
|
+
}),
|
|
1422
|
+
|
|
1423
|
+
/**
|
|
1424
|
+
* Open a live Omni agentic-voice session over `/v1/omni`. Returns an
|
|
1425
|
+
* {@link OmniConnection} that handles the wire protocol's frame-key
|
|
1426
|
+
* asymmetry for you — it sends `type`-keyed control frames (`configure`,
|
|
1427
|
+
* `dtmf`) and parses `event`-keyed server frames — so you can't trip the #1
|
|
1428
|
+
* Omni integration bug. Omni is zero-state: nothing to create first; the
|
|
1429
|
+
* agent's behavior travels in the `configure` frame. Pass `token` (from
|
|
1430
|
+
* `createSession`) to connect from a browser without the secret key.
|
|
1431
|
+
*/
|
|
1432
|
+
connect: (opts: OmniConnectOptions = {}): OmniConnection => {
|
|
1433
|
+
const query: Record<string, string> = { ...(opts.query ?? {}) };
|
|
1434
|
+
if (opts.format) query.format = opts.format;
|
|
1435
|
+
if (opts.rate) query.rate = String(opts.rate);
|
|
1436
|
+
const url = this.realtimeURL({ product: "omni", sessionLabel: opts.sessionLabel, query });
|
|
1437
|
+
const sub = opts.token ? `pyai-key.${opts.token}` : this.realtimeSubprotocol();
|
|
1438
|
+
return new OmniConnection(url, sub, opts);
|
|
1439
|
+
},
|
|
1440
|
+
};
|
|
1441
|
+
|
|
1067
1442
|
// --- realtime (WebSocket) ----------------------------------------------
|
|
1068
1443
|
|
|
1069
1444
|
/** Build the realtime WebSocket URL for the chosen product. */
|
|
@@ -1071,10 +1446,12 @@ export class PyAI {
|
|
|
1071
1446
|
const wsBase = this.baseURL.replace(/^http/, "ws");
|
|
1072
1447
|
const q = new URLSearchParams(opts.query ?? {});
|
|
1073
1448
|
if ((opts.product ?? "omni") === "omni") {
|
|
1074
|
-
// Omni's native realtime surface is /v1/omni.
|
|
1075
|
-
//
|
|
1076
|
-
// connect URL, so default to
|
|
1077
|
-
|
|
1449
|
+
// Omni's native realtime surface is /v1/omni. The session is authorized by
|
|
1450
|
+
// the key's org (zero-state) — sessionLabel is an optional opaque tag.
|
|
1451
|
+
// format/rate are load-bearing on the connect URL, so default to
|
|
1452
|
+
// browser-grade PCM16/24kHz.
|
|
1453
|
+
if (opts.sessionLabel) q.set("session_label", opts.sessionLabel);
|
|
1454
|
+
else if (opts.agentId) q.set("agent_id", opts.agentId); // deprecated alias
|
|
1078
1455
|
if (!q.has("format")) q.set("format", "pcm16");
|
|
1079
1456
|
if (!q.has("rate")) q.set("rate", "24000");
|
|
1080
1457
|
const qs = q.toString();
|
|
@@ -1098,6 +1475,8 @@ export class PyAI {
|
|
|
1098
1475
|
if (opts.sampleRate !== undefined) q.set("sample_rate", String(opts.sampleRate));
|
|
1099
1476
|
if (opts.encoding) q.set("encoding", opts.encoding);
|
|
1100
1477
|
if (opts.interimResults !== undefined) q.set("interim_results", String(opts.interimResults));
|
|
1478
|
+
if (opts.numerals !== undefined) q.set("numerals", String(opts.numerals));
|
|
1479
|
+
if (opts.endpointingMs !== undefined) q.set("endpointing_ms", String(opts.endpointingMs));
|
|
1101
1480
|
const qs = q.toString();
|
|
1102
1481
|
return `${wsBase}/v1/audio/transcriptions/stream${qs ? `?${qs}` : ""}`;
|
|
1103
1482
|
}
|