@pyai/sdk 0.7.2 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -1,3 +1,6 @@
1
+ import { validateSpeechResponse } from "./speech-metadata.ts";
2
+ export { formatSpeechText, type SpeechTextPart } from "./speech-text.ts";
3
+
1
4
  /**
2
5
  * @pyai/sdk, official TypeScript/JavaScript client for the PyAI API.
3
6
  *
@@ -14,8 +17,12 @@ export interface PyAIOptions {
14
17
  baseURL?: string;
15
18
  /** Injectable for tests / custom transports. Defaults to global fetch. */
16
19
  fetch?: typeof fetch;
17
- /** Retries on 429 + 5xx (honors Retry-After). Default 2. */
20
+ /** Retries safe reads on transient failures. Writes are never replayed. Default 2. */
18
21
  maxRetries?: number;
22
+ /** Retry admission window, in ms. Does not time out an in-flight request/body. Default 10000. */
23
+ retryWindowMs?: number;
24
+ /** Optional caller-owned cancellation/deadline signal for REST requests and backoff. */
25
+ signal?: AbortSignal;
19
26
  }
20
27
 
21
28
  /** Stable, machine-readable error. Branch on `code`, not `message`. */
@@ -51,6 +58,8 @@ export interface Voice {
51
58
  available_on?: Array<"speak" | "omni">;
52
59
  /** Accepted Speak delivery modes. Empty means the voice has no Speak surface. */
53
60
  synthesis_modes?: Array<"streaming" | "async">;
61
+ /** Qualified Speak directions. Empty or absent means no directed emotion. */
62
+ speak_emotions?: Array<"neutral" | "happy" | "sad" | "angry" | "fearful" | "surprised">;
54
63
  /** Voice-specific amount on top of the selected product's base rate. */
55
64
  pricing?: {
56
65
  included_in_base_price: boolean;
@@ -148,6 +157,8 @@ export interface SpeechParams {
148
157
  sample_rate?: SpeechSampleRate;
149
158
  /** Reserved; currently returns `400 unsupported_parameter` when provided. */
150
159
  speed?: number;
160
+ /** Named delivery; inspect speak_emotions on the selected catalog voice. */
161
+ emotion?: "neutral" | "happy" | "sad" | "angry" | "fearful" | "surprised";
151
162
  /**
152
163
  * Reserved; currently returns `400 unsupported_parameter` when provided.
153
164
  */
@@ -215,12 +226,17 @@ export interface HearVocabulary {
215
226
  updated_at: number | null;
216
227
  }
217
228
 
229
+ /** Canonical Omni session language identifiers accepted by the public API. */
230
+ export type OmniLanguage = "en" | "fr" | "es" | "de" | "hi" | "zh";
231
+ /** Internal candidate identifiers; the public control plane rejects these until qualified. */
232
+ export type OmniStagedLanguage = "ht" | "zh-CN" | "ar-MSA" | "ar-Gulf";
233
+
218
234
  export interface AgentConfig {
219
235
  name?: string;
220
236
  persona_system_prompt?: string | null;
221
237
  greeting?: string | null;
222
238
  voice_id?: string | null;
223
- language?: "en" | "fr" | "es" | "de" | "hi" | "zh" | null;
239
+ language?: OmniLanguage | null;
224
240
  /**
225
241
  * Opt-in speech-recognition vocabulary. PyAI sanitizes and keeps at most
226
242
  * five terms. An empty list or null turns it off.
@@ -477,7 +493,7 @@ export interface HearStreamOptions {
477
493
  /** Streaming STT model. Server default `pyai-hear`. */
478
494
  model?: string;
479
495
  /** Omit or use auto for automatic detection; an explicit code pins recognition. */
480
- language?: "auto" | "en" | "es" | "fr" | "de" | "hi" | "it" | "pt" | "nl";
496
+ language?: "auto" | "en" | "es" | "fr" | "de" | "hi" | "it" | "pt" | "nl" | "zh" | "yue" | "ar" | "id" | "ko" | "th" | "vi" | "ja" | "tr" | "ms" | "sv" | "da" | "fi" | "pl" | "cs" | "fil" | "fa" | "el" | "hu" | "mk" | "ro";
481
497
  /** Input PCM sample rate in Hz. Default 16000 server-side. */
482
498
  sampleRate?: number;
483
499
  /** Audio frame encoding. Default "pcm16". */
@@ -800,8 +816,12 @@ function omniTranscriptRole(value: unknown): "user" | "assistant" | null {
800
816
  return null;
801
817
  }
802
818
 
803
- /** Normalize the live UTF-8 `0x02` text body and bounded legacy JSON bodies. */
804
- export function normalizeOmniTranscriptBody(bytes: Uint8Array): OmniTranscriptFrame | null {
819
+ /** Transcript encoding is selected by the transport, never inferred from text. */
820
+ export type OmniTranscriptFormat = "text" | "legacy-json";
821
+
822
+ /** Normalize native caller text, or explicitly selected bounded legacy JSON. */
823
+ export function normalizeOmniTranscriptBody(bytes: Uint8Array, format: OmniTranscriptFormat = "text"): OmniTranscriptFrame | null {
824
+ if (format !== "text" && format !== "legacy-json") return null;
805
825
  if (bytes.byteLength === 0 || bytes.byteLength > OMNI_TRANSCRIPT_MAX_BYTES) return null;
806
826
  let decoded: string;
807
827
  try {
@@ -811,14 +831,14 @@ export function normalizeOmniTranscriptBody(bytes: Uint8Array): OmniTranscriptFr
811
831
  }
812
832
  // The serving engine's canonical payload is plain UTF-8 caller text, not
813
833
  // JSON. Each frame is a delta for the current caller turn.
814
- if (!decoded.trimStart().startsWith("{")) {
834
+ if (format === "text") {
815
835
  const text = omniTranscriptText(decoded);
816
836
  return text
817
837
  ? { event: "transcript", role: "user", text, final: false, mode: "delta" }
818
838
  : null;
819
839
  }
820
840
 
821
- // Keep accepting a bounded direct object for older bridges and recordings.
841
+ // Older bridges/recordings must explicitly select their JSON encoding.
822
842
  let value: unknown;
823
843
  try {
824
844
  value = JSON.parse(decoded);
@@ -908,7 +928,20 @@ export interface OmniToolCallFrame {
908
928
  * you, which is the whole point: a hand-rolled `{"event":"configure"}` is
909
929
  * acked but silently dropped, leaving the agent with no brain and zero turns.
910
930
  */
931
+ /** Audio identity and progress in output-rate PCM samples, never wall time. */
932
+ export interface OmniPlaybackChunk {
933
+ replyId: string;
934
+ segmentId: string;
935
+ sampleRate: number;
936
+ endSample: number;
937
+ /** Call only after this chunk actually finishes playing, never on receipt. */
938
+ played: () => void;
939
+ }
940
+
911
941
  export interface OmniConfigure {
942
+ /** Require player receipts before assistant speech enters conversation memory.
943
+ * Confirm support through configured.playback_ack before relying on it. */
944
+ playback_ack?: boolean;
912
945
  /** Voice to speak with (stock / clone / designed id). */
913
946
  voice_id?: string;
914
947
  /** System prompt / role + instructions for the agent. */
@@ -927,7 +960,7 @@ export interface OmniConfigure {
927
960
  * proceeds and bills as what was served. Per-language availability is
928
961
  * staged, see the Language support reference.
929
962
  */
930
- language?: "en" | "fr" | "es" | "de" | "hi" | "zh";
963
+ language?: OmniLanguage;
931
964
  /** Function calling: hosted catalog names, client-loop schemas, or names of
932
965
  * server tools already registered with POST /v1/tools. An inline `endpoint`
933
966
  * is rejected with `unsupported_tool_transport`. */
@@ -937,8 +970,14 @@ export interface OmniConfigure {
937
970
  }
938
971
 
939
972
  export interface OmniConnectOptions {
973
+ /** Native `0x02` caller text by default. Select legacy-json only for a known older JSON transport. */
974
+ transcriptFormat?: OmniTranscriptFormat;
940
975
  /** Optional opaque per-session tag echoed to your `kb_endpoint`. Omni is zero-state. */
941
976
  sessionLabel?: string;
977
+ /** Single-use token from session_started. Failed restoration closes instead of starting fresh. */
978
+ resumeToken?: string;
979
+ /** Maximum wait for resume acknowledgement, including socket opening. Default 5000 ms. */
980
+ resumeTimeoutMs?: number;
942
981
  /**
943
982
  * Ephemeral browser token from `omni.createSession`. When set it is used as
944
983
  * the WS subprotocol (`pyai-key.<token>`) instead of the client's secret key,
@@ -963,7 +1002,7 @@ export interface OmniConnectOptions {
963
1002
  /** Fired once the socket opens (after the optional auto-configure). */
964
1003
  onOpen?: () => void;
965
1004
  /** Fired for each binary agent-audio chunk, play it out as it arrives. */
966
- onAudio?: (chunk: OmniAudioChunk) => void;
1005
+ onAudio?: (chunk: OmniAudioChunk, playback?: OmniPlaybackChunk) => void;
967
1006
  /** Fired on the `hello` handshake frame. */
968
1007
  onHello?: (frame: OmniServerFrame) => void;
969
1008
  /** Fired on the `configured` ack. */
@@ -974,8 +1013,10 @@ export interface OmniConnectOptions {
974
1013
  onTurn?: (frame: OmniServerFrame) => void;
975
1014
  /** Fired for each normalized caller-transcript delta (`0x02` plain UTF-8 live). */
976
1015
  onTranscript?: (frame: OmniServerFrame) => void;
977
- /** Fired on `barge_in` / `flush` (user interrupted). */
1016
+ /** Clear queued audio on `barge_in`, `flush`, `reply_interrupted` or `reply_failed`. */
978
1017
  onBargeIn?: (frame: OmniServerFrame) => void;
1018
+ /** Generation completion is distinct from acknowledged playback completion. */
1019
+ onReplyState?: (frame: OmniServerFrame) => void;
979
1020
  /** Fired when the engine requests a client-loop tool invocation. */
980
1021
  onToolCall?: (frame: OmniToolCallFrame) => void;
981
1022
  /** Fired on `session_end`. */
@@ -1010,13 +1051,34 @@ export class OmniConnection {
1010
1051
  private readonly ws: WebSocketLike;
1011
1052
  private readonly opts: OmniConnectOptions;
1012
1053
  private closed = false;
1054
+ private playbackEnabled = false;
1055
+ private audioReady = false;
1056
+ private readonly inputRate: number;
1057
+ private readonly outputRate: number;
1058
+ private playbackEpoch = 0;
1059
+ private playbackSegment?: { replyId: string; segmentId: string; sampleRate: number; samples: number; confirmed: number; epoch: number; chunks: Map<number, { end: number; played: boolean }> };
1060
+ private awaitingResume = false;
1061
+ private resumeTimer?: ReturnType<typeof setTimeout>;
1013
1062
  /** Serializes async Blob reads in {@link sendAudio} so frames stay ordered. */
1014
1063
  private blobTail: Promise<void> = Promise.resolve();
1064
+ private outboundQueued = 0;
1015
1065
  /** Serializes inbound Blob decoding so browser frames stay ordered. */
1016
1066
  private inboundTail: Promise<void> = Promise.resolve();
1067
+ private inboundQueued = 0;
1017
1068
 
1018
1069
  constructor(url: string, subprotocol: string, opts: OmniConnectOptions) {
1019
1070
  this.opts = opts;
1071
+ const query = new URL(url).searchParams;
1072
+ this.inputRate = Number(query.get("rate") ?? 24000);
1073
+ if (![8000, 16000, 24000].includes(this.inputRate) || (query.get("format") ?? "pcm16") !== "pcm16") {
1074
+ throw new TypeError("Omni requires PCM16 mono input at 8000, 16000 or 24000 Hz");
1075
+ }
1076
+ this.outputRate = this.inputRate === 8000 ? 8000 : 24000;
1077
+ this.awaitingResume = new URL(url).searchParams.has("resume_token");
1078
+ const resumeTimeout = opts.resumeTimeoutMs ?? 5000;
1079
+ if (!Number.isFinite(resumeTimeout) || resumeTimeout <= 0 || resumeTimeout > 30_000) {
1080
+ throw new Error("resumeTimeoutMs must be greater than zero and at most 30000");
1081
+ }
1020
1082
  const WS = opts.webSocket ?? (globalThis as { WebSocket?: WebSocketCtor }).WebSocket;
1021
1083
  if (!WS) {
1022
1084
  throw new Error(
@@ -1024,8 +1086,11 @@ export class OmniConnection {
1024
1086
  );
1025
1087
  }
1026
1088
  this.ws = new WS(url, [REALTIME_SUBPROTOCOL_MARKER, subprotocol]);
1089
+ if (this.awaitingResume) {
1090
+ this.resumeTimer = setTimeout(() => this.rejectResume("Resume acknowledgement timed out"), resumeTimeout);
1091
+ }
1027
1092
  this.ws.onopen = () => {
1028
- if (opts.configure) {
1093
+ if (opts.configure && !this.awaitingResume) {
1029
1094
  try {
1030
1095
  this.configure(opts.configure);
1031
1096
  } catch {
@@ -1038,20 +1103,70 @@ export class OmniConnection {
1038
1103
  const reportDecodeError = (error: unknown) => {
1039
1104
  opts.onError?.(error instanceof Error ? error : new Error("Could not decode Omni frame"));
1040
1105
  };
1041
- if (typeof Blob !== "undefined" && ev.data instanceof Blob) {
1042
- this.inboundTail = this.inboundTail.then(() => this.handleMessage(ev.data)).catch(reportDecodeError);
1106
+ if ((typeof Blob !== "undefined" && ev.data instanceof Blob) || this.inboundQueued > 0) {
1107
+ this.inboundQueued++;
1108
+ this.inboundTail = this.inboundTail.then(() => this.handleMessage(ev.data))
1109
+ .catch(reportDecodeError).finally(() => { this.inboundQueued--; });
1043
1110
  } else {
1044
1111
  void this.handleMessage(ev.data).catch(reportDecodeError);
1045
1112
  }
1046
1113
  };
1047
1114
  this.ws.onerror = (ev) => opts.onError?.(ev instanceof Error ? ev : new Error("WebSocket error"));
1048
1115
  this.ws.onclose = (ev) => {
1116
+ clearTimeout(this.resumeTimer);
1049
1117
  this.closed = true;
1050
1118
  opts.onClose?.(ev.code, ev.reason);
1051
1119
  };
1052
1120
  }
1053
1121
 
1122
+ private rejectResume(message: string): void {
1123
+ if (this.closed) return;
1124
+ this.close(4000, "resume_failed");
1125
+ this.opts.onError?.(new PyAIError(0, message, "resume_failed"));
1126
+ }
1127
+
1128
+ private assertSendable(): void {
1129
+ if (this.closed) throw new Error("Omni connection is closed");
1130
+ if (this.awaitingResume) throw new Error("Wait for a confirmed session resume before sending");
1131
+ }
1132
+
1133
+ private failAudioContract(message: string): void {
1134
+ this.audioReady = false;
1135
+ this.playbackEnabled = false;
1136
+ this.playbackEpoch++;
1137
+ this.playbackSegment = undefined;
1138
+ try {
1139
+ this.opts.onBargeIn?.({ event: "flush", reason: "audio_format_mismatch" });
1140
+ } finally {
1141
+ this.close(4002, "audio_format_mismatch");
1142
+ this.opts.onError?.(new Error(message));
1143
+ }
1144
+ }
1145
+
1146
+ private validateAudioControl(frame: OmniServerFrame): boolean {
1147
+ if (!["hello", "configured", "audio_segment_start"].includes(frame.event)) return true;
1148
+ let valid = frame.event === "hello" || this.audioReady;
1149
+ const expected: Record<string, unknown> = {
1150
+ format: "pcm16", audio_format: "pcm16", audio_out_format: "pcm16",
1151
+ sample_rate: this.outputRate, audio_out_sample_rate: this.outputRate,
1152
+ channels: 1, audio_channels: 1, audio_out_channels: 1,
1153
+ audio_in: `pcm16@${this.inputRate}`, audio_out: `pcm16@${this.outputRate}`,
1154
+ };
1155
+ for (const [key, value] of Object.entries(expected)) {
1156
+ if (key in frame && frame[key] !== value) valid = false;
1157
+ }
1158
+ if (frame.event === "hello" && frame.audio_out !== expected.audio_out) valid = false;
1159
+ if (frame.event === "audio_segment_start" && frame.sample_rate !== this.outputRate) valid = false;
1160
+ if (!valid) {
1161
+ this.failAudioContract("Omni audio declaration does not match the negotiated PCM16 mono format");
1162
+ return false;
1163
+ }
1164
+ if (frame.event === "hello") this.audioReady = true;
1165
+ return true;
1166
+ }
1167
+
1054
1168
  private async handleMessage(data: unknown): Promise<void> {
1169
+ if (this.closed) return;
1055
1170
  // Server → client binary frames are TYPE-TAGGED by their first byte:
1056
1171
  // 0x01 = agent audio (PCM16) · 0x02 = transcript UTF-8 · 0x03 = control JSON.
1057
1172
  // (Treating every binary frame as audio, the old behavior, plays the
@@ -1064,13 +1179,43 @@ export class OmniConnection {
1064
1179
  this.opts.onError?.(new Error("Unsupported Omni binary frame"));
1065
1180
  return;
1066
1181
  }
1182
+ if (this.closed) return;
1067
1183
  const tag = bytes[0];
1184
+ if (this.awaitingResume && tag !== 0x03) return;
1068
1185
  if (tag === 0x01) {
1069
- this.opts.onAudio?.(bytes.slice(1) as OmniAudioChunk); // copy → aligned PCM16
1186
+ if (!this.audioReady || (bytes.length - 1) % 2) {
1187
+ this.failAudioContract("Omni audio requires a valid hello and whole PCM16 samples");
1188
+ return;
1189
+ }
1190
+ const audio = bytes.slice(1) as OmniAudioChunk; // copy → aligned PCM16
1191
+ const segment = this.playbackSegment;
1192
+ let playback: OmniPlaybackChunk | undefined;
1193
+ if (this.playbackEnabled && segment && bytes.length > 1) {
1194
+ const startSample = segment.samples;
1195
+ segment.samples += (bytes.length - 1) / 2;
1196
+ const endSample = segment.samples;
1197
+ const pending = { end: endSample, played: false };
1198
+ segment.chunks.set(startSample, pending);
1199
+ playback = { replyId: segment.replyId, segmentId: segment.segmentId,
1200
+ sampleRate: segment.sampleRate, endSample,
1201
+ played: () => {
1202
+ if (pending.played || this.closed || segment.epoch !== this.playbackEpoch) return;
1203
+ pending.played = true;
1204
+ const before = segment.confirmed;
1205
+ let next = segment.chunks.get(segment.confirmed);
1206
+ while (next?.played) {
1207
+ segment.chunks.delete(segment.confirmed);
1208
+ segment.confirmed = next.end;
1209
+ next = segment.chunks.get(segment.confirmed);
1210
+ }
1211
+ if (segment.confirmed > before) this.playbackProgress(segment.replyId, segment.segmentId, segment.confirmed);
1212
+ } };
1213
+ }
1214
+ this.opts.onAudio?.(audio, playback);
1070
1215
  return;
1071
1216
  }
1072
1217
  if (tag === 0x02) {
1073
- const transcript = normalizeOmniTranscriptBody(bytes.subarray(1));
1218
+ const transcript = normalizeOmniTranscriptBody(bytes.subarray(1), this.opts.transcriptFormat);
1074
1219
  if (transcript) this.dispatchFrame(transcript);
1075
1220
  else this.opts.onError?.(new Error("Unparseable Omni transcript frame"));
1076
1221
  return;
@@ -1078,6 +1223,7 @@ export class OmniConnection {
1078
1223
  if (tag === 0x03) {
1079
1224
  try {
1080
1225
  const parsed = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(1))) as OmniServerFrame;
1226
+ if (this.awaitingResume && parsed?.event !== OmniEvent.Hello && parsed?.event !== OmniEvent.SessionStarted && parsed?.event !== OmniEvent.Error) return;
1081
1227
  if (parsed?.event === OmniEvent.Transcript) {
1082
1228
  // The deployed engine also sends this bounded synthesis-input
1083
1229
  // advisory. It is not caller ASR or proof that speech was played.
@@ -1112,6 +1258,43 @@ export class OmniConnection {
1112
1258
  this.opts.onError?.(new Error("Omni server control frame is missing its event key"));
1113
1259
  return;
1114
1260
  }
1261
+ if (this.closed) return;
1262
+ if (!this.validateAudioControl(frame)) return;
1263
+ if (this.awaitingResume) {
1264
+ if (eventName === OmniEvent.SessionStarted) {
1265
+ if (frame.resumed !== true) {
1266
+ this.rejectResume("The server could not restore the previous conversation");
1267
+ return;
1268
+ }
1269
+ clearTimeout(this.resumeTimer);
1270
+ this.awaitingResume = false;
1271
+ if (this.opts.configure) this.configure(this.opts.configure);
1272
+ } else if (eventName !== OmniEvent.Hello && eventName !== OmniEvent.Error) {
1273
+ return;
1274
+ }
1275
+ }
1276
+ if (eventName === "configured") this.playbackEnabled = frame.playback_ack === true;
1277
+ if (eventName === "audio_segment_start" && this.playbackEnabled) {
1278
+ if (typeof frame.reply_id !== "string" || !/^[a-f0-9]{32}$/.test(frame.reply_id)
1279
+ || typeof frame.segment_id !== "string" || !/^[1-9][0-9]{0,5}$/.test(frame.segment_id)
1280
+ || (frame.sample_rate !== 8000 && frame.sample_rate !== 24000)) {
1281
+ this.failAudioContract("Invalid Omni playback segment");
1282
+ return;
1283
+ }
1284
+ this.playbackSegment = { replyId: frame.reply_id, segmentId: frame.segment_id,
1285
+ sampleRate: frame.sample_rate, samples: 0, confirmed: 0, epoch: this.playbackEpoch, chunks: new Map() };
1286
+ } else if (eventName === "audio_segment_done"
1287
+ && this.playbackSegment?.replyId === frame.reply_id
1288
+ && this.playbackSegment?.segmentId === frame.segment_id) {
1289
+ this.playbackSegment = undefined;
1290
+ } else if (eventName === "flush" || eventName === "barge_in"
1291
+ || eventName === "reply_interrupted" || eventName === "reply_failed") {
1292
+ this.playbackEpoch++;
1293
+ this.playbackSegment = undefined;
1294
+ }
1295
+ if (["reply_generation_done", "reply_playback_done", "reply_interrupted", "reply_failed"].includes(eventName)) {
1296
+ this.opts.onReplyState?.(frame);
1297
+ }
1115
1298
  this.opts.onEvent?.(frame);
1116
1299
  switch (eventName) {
1117
1300
  case OmniEvent.Hello:
@@ -1131,6 +1314,8 @@ export class OmniConnection {
1131
1314
  break;
1132
1315
  case OmniEvent.BargeIn:
1133
1316
  case OmniEvent.Flush:
1317
+ case "reply_interrupted":
1318
+ case "reply_failed":
1134
1319
  this.opts.onBargeIn?.(frame);
1135
1320
  break;
1136
1321
  case OmniEvent.ToolCall:
@@ -1154,10 +1339,11 @@ export class OmniConnection {
1154
1339
  * `{"event":"configure"}` is acked but silently dropped by the engine.)
1155
1340
  */
1156
1341
  configure(cfg: OmniConfigure): void {
1342
+ this.assertSendable();
1157
1343
  const payload: Record<string, unknown> = { ...cfg };
1158
1344
  delete payload.type;
1159
1345
  delete payload.event;
1160
- this.ws.send(omniControlFrame({ type: "configure", ...payload }));
1346
+ this.sendOrdered(omniControlFrame({ type: "configure", ...payload }));
1161
1347
  }
1162
1348
 
1163
1349
  /**
@@ -1166,29 +1352,64 @@ export class OmniConnection {
1166
1352
  * default branch, so an untagged chunk is dropped with no error.
1167
1353
  */
1168
1354
  sendAudio(chunk: ArrayBufferLike | ArrayBufferView | Blob): void {
1355
+ this.assertSendable();
1169
1356
  const bytes = omniToBytes(chunk);
1170
1357
  if (bytes) {
1171
- this.ws.send(omniAudioFrame(bytes));
1358
+ this.sendOrdered(omniAudioFrame(bytes));
1172
1359
  return;
1173
1360
  }
1174
- // A Blob can only be read asynchronously; queue behind the previous read so
1175
- // frames still reach the engine in call order.
1176
- this.blobTail = this.blobTail
1177
- .then(async () => {
1178
- const buf = await (chunk as Blob).arrayBuffer();
1179
- this.ws.send(omniAudioFrame(new Uint8Array(buf)));
1180
- })
1181
- .catch((err) => this.opts.onError?.(err instanceof Error ? err : new Error(String(err))));
1361
+ // Reserve order before reading a Blob: later audio and controls must not
1362
+ // overtake it. Closing still cancels queued work rather than replaying it.
1363
+ this.queueOutbound(async () => omniAudioFrame(new Uint8Array(await (chunk as Blob).arrayBuffer())));
1182
1364
  }
1183
1365
 
1184
- /** Send DTMF digits as a `{"type":"dtmf"}` control frame. */
1366
+ private queueOutbound(frame: () => ArrayBufferLike | ArrayBufferView | Promise<ArrayBufferLike | ArrayBufferView>): void {
1367
+ this.outboundQueued++;
1368
+ this.blobTail = this.blobTail.then(async () => {
1369
+ this.assertSendable();
1370
+ const data = await frame();
1371
+ this.assertSendable();
1372
+ this.ws.send(data);
1373
+ }).catch((error) => {
1374
+ this.opts.onError?.(error instanceof Error ? error : new Error(String(error)));
1375
+ }).finally(() => { this.outboundQueued--; });
1376
+ }
1377
+
1378
+ private sendOrdered(data: ArrayBufferLike | ArrayBufferView): void {
1379
+ if (this.outboundQueued) this.queueOutbound(() => data);
1380
+ else this.ws.send(data);
1381
+ }
1382
+
1383
+ /** Report samples actually rendered by the player in one reply segment.
1384
+ * Receiving or scheduling bytes is not playback. Stale receipts are ignored
1385
+ * server-side; this never changes completed tool actions. */
1386
+ playbackProgress(replyId: string, segmentId: string, playedSamples: number): void {
1387
+ this.assertSendable();
1388
+ if (!this.playbackEnabled) throw new Error("Playback acknowledgments were not negotiated");
1389
+ if (!/^[a-f0-9]{32}$/.test(replyId) || !/^[1-9][0-9]{0,5}$/.test(segmentId)
1390
+ || !Number.isSafeInteger(playedSamples) || playedSamples < 0) {
1391
+ throw new TypeError("Invalid playback progress");
1392
+ }
1393
+ this.sendOrdered(omniControlFrame({ type: "playback_progress", reply_id: replyId,
1394
+ segment_id: segmentId, played_samples: playedSamples }));
1395
+ }
1396
+
1397
+ /** Send a nonempty DTMF sequence as ordered single-key control frames.
1398
+ * Valid keys are 0–9, *, # and A–D; lowercase a–d are normalized. */
1185
1399
  sendDtmf(digits: string): void {
1186
- this.ws.send(omniControlFrame({ type: "dtmf", digits }));
1400
+ this.assertSendable();
1401
+ if (typeof digits !== "string" || digits.length === 0 || /[^0-9*#A-Da-d]/.test(digits)) {
1402
+ throw new TypeError("Invalid DTMF sequence");
1403
+ }
1404
+ for (const digit of digits.toUpperCase()) {
1405
+ this.sendOrdered(omniControlFrame({ type: "dtmf", digit }));
1406
+ }
1187
1407
  }
1188
1408
 
1189
1409
  /** Reply to a client-loop {@link OmniEvent.ToolCall}. */
1190
1410
  toolResult(callId: string, payload: { result?: unknown; error?: string }): void {
1191
- this.ws.send(omniControlFrame({ type: "tool_result", call_id: callId, ...payload }));
1411
+ this.assertSendable();
1412
+ this.sendOrdered(omniControlFrame({ type: "tool_result", call_id: callId, ...payload }));
1192
1413
  }
1193
1414
 
1194
1415
  /**
@@ -1197,17 +1418,22 @@ export class OmniConnection {
1197
1418
  * never `event`.
1198
1419
  */
1199
1420
  send(frame: Record<string, unknown>): void {
1421
+ this.assertSendable();
1200
1422
  if (typeof frame.type !== "string" || !frame.type) {
1201
1423
  throw new TypeError("Omni client control frames must have a non-empty type key");
1202
1424
  }
1203
1425
  const payload = { ...frame };
1204
1426
  delete payload.event;
1205
- this.ws.send(omniControlFrame(payload));
1427
+ this.sendOrdered(omniControlFrame(payload));
1206
1428
  }
1207
1429
 
1208
1430
  /** Close the session. */
1209
1431
  close(code: number = WSCloseCode.Normal, reason = ""): void {
1210
- if (!this.closed) this.ws.close(code, reason);
1432
+ clearTimeout(this.resumeTimer);
1433
+ if (!this.closed) {
1434
+ this.closed = true;
1435
+ this.ws.close(code, reason);
1436
+ }
1211
1437
  }
1212
1438
 
1213
1439
  /** The underlying socket (escape hatch for advanced use). */
@@ -1660,11 +1886,43 @@ export interface AmdStreamOptions {
1660
1886
 
1661
1887
  const RETRYABLE = new Set([429, 500, 502, 503, 504]);
1662
1888
 
1889
+ // Only transport codes known to be transient; do not retry TLS, URL or DNS
1890
+ // configuration errors. Browsers often hide the cause: those propagate intact.
1891
+ const TRANSIENT_TRANSPORT = new Set(["ECONNRESET", "ECONNREFUSED", "EPIPE", "ETIMEDOUT", "EAI_AGAIN", "UND_ERR_SOCKET", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT", "UND_ERR_BODY_TIMEOUT"]);
1892
+ function transientTransport(error: unknown): boolean {
1893
+ const seen = new Set<unknown>();
1894
+ while (error && typeof error === "object" && !seen.has(error)) {
1895
+ seen.add(error);
1896
+ const value = error as { name?: string; code?: string; cause?: unknown };
1897
+ if (value.name === "AbortError" || value.name === "TimeoutError") return false;
1898
+ if (value.code && TRANSIENT_TRANSPORT.has(value.code)) return true;
1899
+ error = value.cause;
1900
+ }
1901
+ return false;
1902
+ }
1903
+ function retryDelay(header: string | null, attempt: number): number {
1904
+ const jitter = Math.random() * Math.min(2 ** attempt * 250, 2000);
1905
+ if (header === null) return jitter;
1906
+ const seconds = /^\d+(?:\.\d+)?$/.test(header.trim()) ? Number(header) : NaN;
1907
+ const delay = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(header) - Date.now();
1908
+ return Number.isFinite(delay) ? Math.max(0, delay) + jitter : jitter;
1909
+ }
1910
+ function retryPause(ms: number, signal?: AbortSignal | null): Promise<void> {
1911
+ return new Promise((resolve, reject) => {
1912
+ const cancel = () => { clearTimeout(timer); signal?.removeEventListener("abort", cancel); reject(signal?.reason); };
1913
+ const timer = setTimeout(() => { signal?.removeEventListener("abort", cancel); resolve(); }, ms);
1914
+ signal?.addEventListener("abort", cancel, { once: true });
1915
+ if (signal?.aborted) cancel();
1916
+ });
1917
+ }
1918
+
1663
1919
  export class PyAI {
1664
1920
  private readonly apiKey: string;
1665
1921
  private readonly baseURL: string;
1666
1922
  private readonly fetchImpl: typeof fetch;
1667
1923
  private readonly maxRetries: number;
1924
+ private readonly retryWindowMs: number;
1925
+ private readonly signal?: AbortSignal;
1668
1926
 
1669
1927
  constructor(opts: PyAIOptions) {
1670
1928
  if (!opts.apiKey) throw new Error("apiKey is required");
@@ -1672,6 +1930,10 @@ export class PyAI {
1672
1930
  this.baseURL = (opts.baseURL ?? "https://api.pyai.com").replace(/\/$/, "");
1673
1931
  this.fetchImpl = opts.fetch ?? globalThis.fetch;
1674
1932
  this.maxRetries = opts.maxRetries ?? 2;
1933
+ this.retryWindowMs = opts.retryWindowMs ?? 10_000;
1934
+ this.signal = opts.signal;
1935
+ if (!Number.isSafeInteger(this.maxRetries) || this.maxRetries < 0) throw new Error("maxRetries must be a non-negative integer");
1936
+ if (!Number.isFinite(this.retryWindowMs) || this.retryWindowMs < 0) throw new Error("retryWindowMs must be finite and non-negative");
1675
1937
  if (!this.fetchImpl) throw new Error("No fetch implementation available; pass opts.fetch");
1676
1938
  }
1677
1939
 
@@ -1681,24 +1943,50 @@ export class PyAI {
1681
1943
  return { Authorization: `Bearer ${this.apiKey}`, ...extra };
1682
1944
  }
1683
1945
 
1684
- private async request(path: string, init: RequestInit, attempt = 0): Promise<Response> {
1685
- const res = await this.fetchImpl(`${this.baseURL}${path}`, init);
1686
- if (res.ok) return res;
1687
- if (RETRYABLE.has(res.status) && attempt < this.maxRetries) {
1688
- const retryAfter = Number(res.headers.get("retry-after"));
1689
- const delayMs = Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter * 1000 : 2 ** attempt * 250;
1690
- await new Promise((r) => setTimeout(r, delayMs));
1691
- return this.request(path, init, attempt + 1);
1946
+ private async request(path: string, init: RequestInit): Promise<Response> {
1947
+ const signal = init.signal ?? this.signal;
1948
+ const safe = ["GET", "HEAD"].includes((init.method ?? "GET").toUpperCase());
1949
+ const deadline = performance.now() + this.retryWindowMs;
1950
+ for (let attempt = 0; ; attempt++) {
1951
+ signal?.throwIfAborted();
1952
+ let res: Response | undefined;
1953
+ let failure: unknown;
1954
+ try {
1955
+ res = await this.fetchImpl(`${this.baseURL}${path}`, { ...init, signal });
1956
+ } catch (error) {
1957
+ // Preserve the original exception and cause; an arbitrary TypeError may
1958
+ // be a programming/CORS/configuration error, not a transient connection.
1959
+ if (signal?.aborted || !transientTransport(error)) throw error;
1960
+ failure = error;
1961
+ }
1962
+ if (res?.ok) return res;
1963
+ if (!safe || attempt >= this.maxRetries || (res && !RETRYABLE.has(res.status))) {
1964
+ if (res) throw await this.toError(res, signal);
1965
+ throw failure;
1966
+ }
1967
+ const delay = retryDelay(res?.headers.get("retry-after") ?? null, attempt);
1968
+ if (performance.now() + delay >= deadline) {
1969
+ if (res) throw await this.toError(res, signal);
1970
+ throw failure;
1971
+ }
1972
+ // Dispose the failed response before waiting/reusing its connection.
1973
+ if (res?.body) await res.body.cancel().catch(() => {});
1974
+ await retryPause(delay, signal);
1975
+ if (performance.now() >= deadline) {
1976
+ if (res) throw new PyAIError(res.status, `HTTP ${res.status}`, undefined, undefined, res.headers.get("x-request-id") ?? undefined);
1977
+ throw failure;
1978
+ }
1692
1979
  }
1693
- throw await this.toError(res);
1694
1980
  }
1695
1981
 
1696
- private async toError(res: Response): Promise<PyAIError> {
1982
+ private async toError(res: Response, signal?: AbortSignal | null): Promise<PyAIError> {
1697
1983
  const requestId = res.headers.get("x-request-id") ?? undefined;
1698
1984
  let body: unknown;
1699
1985
  try {
1700
1986
  body = await res.json();
1987
+ signal?.throwIfAborted();
1701
1988
  } catch {
1989
+ signal?.throwIfAborted();
1702
1990
  return new PyAIError(res.status, `HTTP ${res.status}`, undefined, undefined, requestId);
1703
1991
  }
1704
1992
  const b = body as {
@@ -1830,6 +2118,7 @@ export class PyAI {
1830
2118
  headers: this.authHeaders({ "Content-Type": "application/json" }),
1831
2119
  body: JSON.stringify({ model: "pyai-speak", ...params }),
1832
2120
  });
2121
+ await validateSpeechResponse(res, params);
1833
2122
  return res.arrayBuffer();
1834
2123
  },
1835
2124
  /**
@@ -1847,6 +2136,7 @@ export class PyAI {
1847
2136
  headers: this.authHeaders({ "Content-Type": "application/json" }),
1848
2137
  body: JSON.stringify({ model: "pyai-speak", ...params, stream: true }),
1849
2138
  });
2139
+ await validateSpeechResponse(res, params);
1850
2140
  if (!res.body) throw new PyAIError(res.status, "Response had no body to stream");
1851
2141
  return res.body as ReadableStream<Uint8Array>;
1852
2142
  },
@@ -1857,7 +2147,7 @@ export class PyAI {
1857
2147
  filename?: string;
1858
2148
  model?: string;
1859
2149
  /** Optional language hint. Omission enables automatic detection. */
1860
- language?: "en" | "es" | "fr" | "de" | "hi" | "it" | "pt" | "nl";
2150
+ language?: "en" | "es" | "fr" | "de" | "hi" | "it" | "pt" | "nl" | "zh" | "yue" | "ar" | "id" | "ko" | "th" | "vi" | "ja" | "tr" | "ms" | "sv" | "da" | "fi" | "pl" | "cs" | "fil" | "fa" | "el" | "hu" | "mk" | "ro";
1861
2151
  numerals?: boolean;
1862
2152
  smart_format?: boolean;
1863
2153
  dictation?: boolean;
@@ -2177,6 +2467,10 @@ export class PyAI {
2177
2467
  */
2178
2468
  connect: (opts: OmniConnectOptions = {}): OmniConnection => {
2179
2469
  const query: Record<string, string> = { ...(opts.query ?? {}) };
2470
+ if (opts.resumeToken !== undefined) {
2471
+ if (!opts.resumeToken.trim()) throw new Error("resumeToken must not be empty");
2472
+ query.resume_token = opts.resumeToken;
2473
+ }
2180
2474
  if (opts.format) query.format = opts.format;
2181
2475
  if (opts.rate) query.rate = String(opts.rate);
2182
2476
  const url = this.realtimeURL({ sessionLabel: opts.sessionLabel, query });