@pyai/sdk 0.8.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -8
- package/dist/cli-omni-template.d.ts +2 -2
- package/dist/cli-omni-template.js +2 -2
- package/dist/index.d.ts +48 -8
- package/dist/index.js +173 -30
- package/dist/speech-metadata.d.ts +5 -0
- package/dist/speech-metadata.js +37 -0
- package/dist/speech-text.d.ts +18 -0
- package/dist/speech-text.js +75 -0
- package/package.json +1 -1
- package/src/cli-omni-template.ts +2 -2
- package/src/index.ts +193 -28
- package/src/speech-metadata.ts +33 -0
- package/src/speech-text.ts +71 -0
package/src/index.ts
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { validateSpeechResponse } from "./speech-metadata.ts";
|
|
2
|
+
export { formatSpeechText, type SpeechTextPart } from "./speech-text.ts";
|
|
3
|
+
|
|
1
4
|
/**
|
|
2
5
|
* @pyai/sdk, official TypeScript/JavaScript client for the PyAI API.
|
|
3
6
|
*
|
|
@@ -55,6 +58,8 @@ export interface Voice {
|
|
|
55
58
|
available_on?: Array<"speak" | "omni">;
|
|
56
59
|
/** Accepted Speak delivery modes. Empty means the voice has no Speak surface. */
|
|
57
60
|
synthesis_modes?: Array<"streaming" | "async">;
|
|
61
|
+
/** Qualified Speak directions. Empty or absent means no directed emotion. */
|
|
62
|
+
speak_emotions?: Array<"neutral" | "happy" | "sad" | "angry" | "fearful" | "surprised">;
|
|
58
63
|
/** Voice-specific amount on top of the selected product's base rate. */
|
|
59
64
|
pricing?: {
|
|
60
65
|
included_in_base_price: boolean;
|
|
@@ -152,6 +157,8 @@ export interface SpeechParams {
|
|
|
152
157
|
sample_rate?: SpeechSampleRate;
|
|
153
158
|
/** Reserved; currently returns `400 unsupported_parameter` when provided. */
|
|
154
159
|
speed?: number;
|
|
160
|
+
/** Named delivery; inspect speak_emotions on the selected catalog voice. */
|
|
161
|
+
emotion?: "neutral" | "happy" | "sad" | "angry" | "fearful" | "surprised";
|
|
155
162
|
/**
|
|
156
163
|
* Reserved; currently returns `400 unsupported_parameter` when provided.
|
|
157
164
|
*/
|
|
@@ -220,7 +227,7 @@ export interface HearVocabulary {
|
|
|
220
227
|
}
|
|
221
228
|
|
|
222
229
|
/** Canonical Omni session language identifiers accepted by the public API. */
|
|
223
|
-
export type OmniLanguage = "en" | "fr" | "es" | "de" | "hi" | "zh";
|
|
230
|
+
export type OmniLanguage = "en" | "fr" | "es" | "de" | "hi" | "zh" | "it" | "pt" | "ja" | "ko";
|
|
224
231
|
/** Internal candidate identifiers; the public control plane rejects these until qualified. */
|
|
225
232
|
export type OmniStagedLanguage = "ht" | "zh-CN" | "ar-MSA" | "ar-Gulf";
|
|
226
233
|
|
|
@@ -486,7 +493,7 @@ export interface HearStreamOptions {
|
|
|
486
493
|
/** Streaming STT model. Server default `pyai-hear`. */
|
|
487
494
|
model?: string;
|
|
488
495
|
/** Omit or use auto for automatic detection; an explicit code pins recognition. */
|
|
489
|
-
language?: "auto" | "en" | "es" | "fr" | "de" | "hi" | "it" | "pt" | "nl";
|
|
496
|
+
language?: "auto" | "en" | "es" | "fr" | "de" | "hi" | "it" | "pt" | "nl" | "zh" | "yue" | "ar" | "id" | "ko" | "th" | "vi" | "ja" | "tr" | "ms" | "sv" | "da" | "fi" | "pl" | "cs" | "fil" | "fa" | "el" | "hu" | "mk" | "ro";
|
|
490
497
|
/** Input PCM sample rate in Hz. Default 16000 server-side. */
|
|
491
498
|
sampleRate?: number;
|
|
492
499
|
/** Audio frame encoding. Default "pcm16". */
|
|
@@ -809,8 +816,12 @@ function omniTranscriptRole(value: unknown): "user" | "assistant" | null {
|
|
|
809
816
|
return null;
|
|
810
817
|
}
|
|
811
818
|
|
|
812
|
-
/**
|
|
813
|
-
export
|
|
819
|
+
/** Transcript encoding is selected by the transport, never inferred from text. */
|
|
820
|
+
export type OmniTranscriptFormat = "text" | "legacy-json";
|
|
821
|
+
|
|
822
|
+
/** Normalize native caller text, or explicitly selected bounded legacy JSON. */
|
|
823
|
+
export function normalizeOmniTranscriptBody(bytes: Uint8Array, format: OmniTranscriptFormat = "text"): OmniTranscriptFrame | null {
|
|
824
|
+
if (format !== "text" && format !== "legacy-json") return null;
|
|
814
825
|
if (bytes.byteLength === 0 || bytes.byteLength > OMNI_TRANSCRIPT_MAX_BYTES) return null;
|
|
815
826
|
let decoded: string;
|
|
816
827
|
try {
|
|
@@ -820,14 +831,14 @@ export function normalizeOmniTranscriptBody(bytes: Uint8Array): OmniTranscriptFr
|
|
|
820
831
|
}
|
|
821
832
|
// The serving engine's canonical payload is plain UTF-8 caller text, not
|
|
822
833
|
// JSON. Each frame is a delta for the current caller turn.
|
|
823
|
-
if (
|
|
834
|
+
if (format === "text") {
|
|
824
835
|
const text = omniTranscriptText(decoded);
|
|
825
836
|
return text
|
|
826
837
|
? { event: "transcript", role: "user", text, final: false, mode: "delta" }
|
|
827
838
|
: null;
|
|
828
839
|
}
|
|
829
840
|
|
|
830
|
-
//
|
|
841
|
+
// Older bridges/recordings must explicitly select their JSON encoding.
|
|
831
842
|
let value: unknown;
|
|
832
843
|
try {
|
|
833
844
|
value = JSON.parse(decoded);
|
|
@@ -917,7 +928,20 @@ export interface OmniToolCallFrame {
|
|
|
917
928
|
* you, which is the whole point: a hand-rolled `{"event":"configure"}` is
|
|
918
929
|
* acked but silently dropped, leaving the agent with no brain and zero turns.
|
|
919
930
|
*/
|
|
931
|
+
/** Audio identity and progress in output-rate PCM samples, never wall time. */
|
|
932
|
+
export interface OmniPlaybackChunk {
|
|
933
|
+
replyId: string;
|
|
934
|
+
segmentId: string;
|
|
935
|
+
sampleRate: number;
|
|
936
|
+
endSample: number;
|
|
937
|
+
/** Call only after this chunk actually finishes playing, never on receipt. */
|
|
938
|
+
played: () => void;
|
|
939
|
+
}
|
|
940
|
+
|
|
920
941
|
export interface OmniConfigure {
|
|
942
|
+
/** Require player receipts before assistant speech enters conversation memory.
|
|
943
|
+
* Confirm support through configured.playback_ack before relying on it. */
|
|
944
|
+
playback_ack?: boolean;
|
|
921
945
|
/** Voice to speak with (stock / clone / designed id). */
|
|
922
946
|
voice_id?: string;
|
|
923
947
|
/** System prompt / role + instructions for the agent. */
|
|
@@ -946,6 +970,8 @@ export interface OmniConfigure {
|
|
|
946
970
|
}
|
|
947
971
|
|
|
948
972
|
export interface OmniConnectOptions {
|
|
973
|
+
/** Native `0x02` caller text by default. Select legacy-json only for a known older JSON transport. */
|
|
974
|
+
transcriptFormat?: OmniTranscriptFormat;
|
|
949
975
|
/** Optional opaque per-session tag echoed to your `kb_endpoint`. Omni is zero-state. */
|
|
950
976
|
sessionLabel?: string;
|
|
951
977
|
/** Single-use token from session_started. Failed restoration closes instead of starting fresh. */
|
|
@@ -976,7 +1002,7 @@ export interface OmniConnectOptions {
|
|
|
976
1002
|
/** Fired once the socket opens (after the optional auto-configure). */
|
|
977
1003
|
onOpen?: () => void;
|
|
978
1004
|
/** Fired for each binary agent-audio chunk, play it out as it arrives. */
|
|
979
|
-
onAudio?: (chunk: OmniAudioChunk) => void;
|
|
1005
|
+
onAudio?: (chunk: OmniAudioChunk, playback?: OmniPlaybackChunk) => void;
|
|
980
1006
|
/** Fired on the `hello` handshake frame. */
|
|
981
1007
|
onHello?: (frame: OmniServerFrame) => void;
|
|
982
1008
|
/** Fired on the `configured` ack. */
|
|
@@ -987,8 +1013,10 @@ export interface OmniConnectOptions {
|
|
|
987
1013
|
onTurn?: (frame: OmniServerFrame) => void;
|
|
988
1014
|
/** Fired for each normalized caller-transcript delta (`0x02` plain UTF-8 live). */
|
|
989
1015
|
onTranscript?: (frame: OmniServerFrame) => void;
|
|
990
|
-
/**
|
|
1016
|
+
/** Clear queued audio on `barge_in`, `flush`, `reply_interrupted` or `reply_failed`. */
|
|
991
1017
|
onBargeIn?: (frame: OmniServerFrame) => void;
|
|
1018
|
+
/** Generation completion is distinct from acknowledged playback completion. */
|
|
1019
|
+
onReplyState?: (frame: OmniServerFrame) => void;
|
|
992
1020
|
/** Fired when the engine requests a client-loop tool invocation. */
|
|
993
1021
|
onToolCall?: (frame: OmniToolCallFrame) => void;
|
|
994
1022
|
/** Fired on `session_end`. */
|
|
@@ -1023,15 +1051,29 @@ export class OmniConnection {
|
|
|
1023
1051
|
private readonly ws: WebSocketLike;
|
|
1024
1052
|
private readonly opts: OmniConnectOptions;
|
|
1025
1053
|
private closed = false;
|
|
1054
|
+
private playbackEnabled = false;
|
|
1055
|
+
private audioReady = false;
|
|
1056
|
+
private readonly inputRate: number;
|
|
1057
|
+
private readonly outputRate: number;
|
|
1058
|
+
private playbackEpoch = 0;
|
|
1059
|
+
private playbackSegment?: { replyId: string; segmentId: string; sampleRate: number; samples: number; confirmed: number; epoch: number; chunks: Map<number, { end: number; played: boolean }> };
|
|
1026
1060
|
private awaitingResume = false;
|
|
1027
1061
|
private resumeTimer?: ReturnType<typeof setTimeout>;
|
|
1028
1062
|
/** Serializes async Blob reads in {@link sendAudio} so frames stay ordered. */
|
|
1029
1063
|
private blobTail: Promise<void> = Promise.resolve();
|
|
1064
|
+
private outboundQueued = 0;
|
|
1030
1065
|
/** Serializes inbound Blob decoding so browser frames stay ordered. */
|
|
1031
1066
|
private inboundTail: Promise<void> = Promise.resolve();
|
|
1067
|
+
private inboundQueued = 0;
|
|
1032
1068
|
|
|
1033
1069
|
constructor(url: string, subprotocol: string, opts: OmniConnectOptions) {
|
|
1034
1070
|
this.opts = opts;
|
|
1071
|
+
const query = new URL(url).searchParams;
|
|
1072
|
+
this.inputRate = Number(query.get("rate") ?? 24000);
|
|
1073
|
+
if (![8000, 16000, 24000].includes(this.inputRate) || (query.get("format") ?? "pcm16") !== "pcm16") {
|
|
1074
|
+
throw new TypeError("Omni requires PCM16 mono input at 8000, 16000 or 24000 Hz");
|
|
1075
|
+
}
|
|
1076
|
+
this.outputRate = this.inputRate === 8000 ? 8000 : 24000;
|
|
1035
1077
|
this.awaitingResume = new URL(url).searchParams.has("resume_token");
|
|
1036
1078
|
const resumeTimeout = opts.resumeTimeoutMs ?? 5000;
|
|
1037
1079
|
if (!Number.isFinite(resumeTimeout) || resumeTimeout <= 0 || resumeTimeout > 30_000) {
|
|
@@ -1061,8 +1103,10 @@ export class OmniConnection {
|
|
|
1061
1103
|
const reportDecodeError = (error: unknown) => {
|
|
1062
1104
|
opts.onError?.(error instanceof Error ? error : new Error("Could not decode Omni frame"));
|
|
1063
1105
|
};
|
|
1064
|
-
if (typeof Blob !== "undefined" && ev.data instanceof Blob) {
|
|
1065
|
-
this.
|
|
1106
|
+
if ((typeof Blob !== "undefined" && ev.data instanceof Blob) || this.inboundQueued > 0) {
|
|
1107
|
+
this.inboundQueued++;
|
|
1108
|
+
this.inboundTail = this.inboundTail.then(() => this.handleMessage(ev.data))
|
|
1109
|
+
.catch(reportDecodeError).finally(() => { this.inboundQueued--; });
|
|
1066
1110
|
} else {
|
|
1067
1111
|
void this.handleMessage(ev.data).catch(reportDecodeError);
|
|
1068
1112
|
}
|
|
@@ -1086,6 +1130,41 @@ export class OmniConnection {
|
|
|
1086
1130
|
if (this.awaitingResume) throw new Error("Wait for a confirmed session resume before sending");
|
|
1087
1131
|
}
|
|
1088
1132
|
|
|
1133
|
+
private failAudioContract(message: string): void {
|
|
1134
|
+
this.audioReady = false;
|
|
1135
|
+
this.playbackEnabled = false;
|
|
1136
|
+
this.playbackEpoch++;
|
|
1137
|
+
this.playbackSegment = undefined;
|
|
1138
|
+
try {
|
|
1139
|
+
this.opts.onBargeIn?.({ event: "flush", reason: "audio_format_mismatch" });
|
|
1140
|
+
} finally {
|
|
1141
|
+
this.close(4002, "audio_format_mismatch");
|
|
1142
|
+
this.opts.onError?.(new Error(message));
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
|
|
1146
|
+
private validateAudioControl(frame: OmniServerFrame): boolean {
|
|
1147
|
+
if (!["hello", "configured", "audio_segment_start"].includes(frame.event)) return true;
|
|
1148
|
+
let valid = frame.event === "hello" || this.audioReady;
|
|
1149
|
+
const expected: Record<string, unknown> = {
|
|
1150
|
+
format: "pcm16", audio_format: "pcm16", audio_out_format: "pcm16",
|
|
1151
|
+
sample_rate: this.outputRate, audio_out_sample_rate: this.outputRate,
|
|
1152
|
+
channels: 1, audio_channels: 1, audio_out_channels: 1,
|
|
1153
|
+
audio_in: `pcm16@${this.inputRate}`, audio_out: `pcm16@${this.outputRate}`,
|
|
1154
|
+
};
|
|
1155
|
+
for (const [key, value] of Object.entries(expected)) {
|
|
1156
|
+
if (key in frame && frame[key] !== value) valid = false;
|
|
1157
|
+
}
|
|
1158
|
+
if (frame.event === "hello" && frame.audio_out !== expected.audio_out) valid = false;
|
|
1159
|
+
if (frame.event === "audio_segment_start" && frame.sample_rate !== this.outputRate) valid = false;
|
|
1160
|
+
if (!valid) {
|
|
1161
|
+
this.failAudioContract("Omni audio declaration does not match the negotiated PCM16 mono format");
|
|
1162
|
+
return false;
|
|
1163
|
+
}
|
|
1164
|
+
if (frame.event === "hello") this.audioReady = true;
|
|
1165
|
+
return true;
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1089
1168
|
private async handleMessage(data: unknown): Promise<void> {
|
|
1090
1169
|
if (this.closed) return;
|
|
1091
1170
|
// Server → client binary frames are TYPE-TAGGED by their first byte:
|
|
@@ -1104,11 +1183,39 @@ export class OmniConnection {
|
|
|
1104
1183
|
const tag = bytes[0];
|
|
1105
1184
|
if (this.awaitingResume && tag !== 0x03) return;
|
|
1106
1185
|
if (tag === 0x01) {
|
|
1107
|
-
this.
|
|
1186
|
+
if (!this.audioReady || (bytes.length - 1) % 2) {
|
|
1187
|
+
this.failAudioContract("Omni audio requires a valid hello and whole PCM16 samples");
|
|
1188
|
+
return;
|
|
1189
|
+
}
|
|
1190
|
+
const audio = bytes.slice(1) as OmniAudioChunk; // copy → aligned PCM16
|
|
1191
|
+
const segment = this.playbackSegment;
|
|
1192
|
+
let playback: OmniPlaybackChunk | undefined;
|
|
1193
|
+
if (this.playbackEnabled && segment && bytes.length > 1) {
|
|
1194
|
+
const startSample = segment.samples;
|
|
1195
|
+
segment.samples += (bytes.length - 1) / 2;
|
|
1196
|
+
const endSample = segment.samples;
|
|
1197
|
+
const pending = { end: endSample, played: false };
|
|
1198
|
+
segment.chunks.set(startSample, pending);
|
|
1199
|
+
playback = { replyId: segment.replyId, segmentId: segment.segmentId,
|
|
1200
|
+
sampleRate: segment.sampleRate, endSample,
|
|
1201
|
+
played: () => {
|
|
1202
|
+
if (pending.played || this.closed || segment.epoch !== this.playbackEpoch) return;
|
|
1203
|
+
pending.played = true;
|
|
1204
|
+
const before = segment.confirmed;
|
|
1205
|
+
let next = segment.chunks.get(segment.confirmed);
|
|
1206
|
+
while (next?.played) {
|
|
1207
|
+
segment.chunks.delete(segment.confirmed);
|
|
1208
|
+
segment.confirmed = next.end;
|
|
1209
|
+
next = segment.chunks.get(segment.confirmed);
|
|
1210
|
+
}
|
|
1211
|
+
if (segment.confirmed > before) this.playbackProgress(segment.replyId, segment.segmentId, segment.confirmed);
|
|
1212
|
+
} };
|
|
1213
|
+
}
|
|
1214
|
+
this.opts.onAudio?.(audio, playback);
|
|
1108
1215
|
return;
|
|
1109
1216
|
}
|
|
1110
1217
|
if (tag === 0x02) {
|
|
1111
|
-
const transcript = normalizeOmniTranscriptBody(bytes.subarray(1));
|
|
1218
|
+
const transcript = normalizeOmniTranscriptBody(bytes.subarray(1), this.opts.transcriptFormat);
|
|
1112
1219
|
if (transcript) this.dispatchFrame(transcript);
|
|
1113
1220
|
else this.opts.onError?.(new Error("Unparseable Omni transcript frame"));
|
|
1114
1221
|
return;
|
|
@@ -1152,6 +1259,7 @@ export class OmniConnection {
|
|
|
1152
1259
|
return;
|
|
1153
1260
|
}
|
|
1154
1261
|
if (this.closed) return;
|
|
1262
|
+
if (!this.validateAudioControl(frame)) return;
|
|
1155
1263
|
if (this.awaitingResume) {
|
|
1156
1264
|
if (eventName === OmniEvent.SessionStarted) {
|
|
1157
1265
|
if (frame.resumed !== true) {
|
|
@@ -1165,6 +1273,28 @@ export class OmniConnection {
|
|
|
1165
1273
|
return;
|
|
1166
1274
|
}
|
|
1167
1275
|
}
|
|
1276
|
+
if (eventName === "configured") this.playbackEnabled = frame.playback_ack === true;
|
|
1277
|
+
if (eventName === "audio_segment_start" && this.playbackEnabled) {
|
|
1278
|
+
if (typeof frame.reply_id !== "string" || !/^[a-f0-9]{32}$/.test(frame.reply_id)
|
|
1279
|
+
|| typeof frame.segment_id !== "string" || !/^[1-9][0-9]{0,5}$/.test(frame.segment_id)
|
|
1280
|
+
|| (frame.sample_rate !== 8000 && frame.sample_rate !== 24000)) {
|
|
1281
|
+
this.failAudioContract("Invalid Omni playback segment");
|
|
1282
|
+
return;
|
|
1283
|
+
}
|
|
1284
|
+
this.playbackSegment = { replyId: frame.reply_id, segmentId: frame.segment_id,
|
|
1285
|
+
sampleRate: frame.sample_rate, samples: 0, confirmed: 0, epoch: this.playbackEpoch, chunks: new Map() };
|
|
1286
|
+
} else if (eventName === "audio_segment_done"
|
|
1287
|
+
&& this.playbackSegment?.replyId === frame.reply_id
|
|
1288
|
+
&& this.playbackSegment?.segmentId === frame.segment_id) {
|
|
1289
|
+
this.playbackSegment = undefined;
|
|
1290
|
+
} else if (eventName === "flush" || eventName === "barge_in"
|
|
1291
|
+
|| eventName === "reply_interrupted" || eventName === "reply_failed") {
|
|
1292
|
+
this.playbackEpoch++;
|
|
1293
|
+
this.playbackSegment = undefined;
|
|
1294
|
+
}
|
|
1295
|
+
if (["reply_generation_done", "reply_playback_done", "reply_interrupted", "reply_failed"].includes(eventName)) {
|
|
1296
|
+
this.opts.onReplyState?.(frame);
|
|
1297
|
+
}
|
|
1168
1298
|
this.opts.onEvent?.(frame);
|
|
1169
1299
|
switch (eventName) {
|
|
1170
1300
|
case OmniEvent.Hello:
|
|
@@ -1184,6 +1314,8 @@ export class OmniConnection {
|
|
|
1184
1314
|
break;
|
|
1185
1315
|
case OmniEvent.BargeIn:
|
|
1186
1316
|
case OmniEvent.Flush:
|
|
1317
|
+
case "reply_interrupted":
|
|
1318
|
+
case "reply_failed":
|
|
1187
1319
|
this.opts.onBargeIn?.(frame);
|
|
1188
1320
|
break;
|
|
1189
1321
|
case OmniEvent.ToolCall:
|
|
@@ -1211,7 +1343,7 @@ export class OmniConnection {
|
|
|
1211
1343
|
const payload: Record<string, unknown> = { ...cfg };
|
|
1212
1344
|
delete payload.type;
|
|
1213
1345
|
delete payload.event;
|
|
1214
|
-
this.
|
|
1346
|
+
this.sendOrdered(omniControlFrame({ type: "configure", ...payload }));
|
|
1215
1347
|
}
|
|
1216
1348
|
|
|
1217
1349
|
/**
|
|
@@ -1223,30 +1355,61 @@ export class OmniConnection {
|
|
|
1223
1355
|
this.assertSendable();
|
|
1224
1356
|
const bytes = omniToBytes(chunk);
|
|
1225
1357
|
if (bytes) {
|
|
1226
|
-
this.
|
|
1358
|
+
this.sendOrdered(omniAudioFrame(bytes));
|
|
1227
1359
|
return;
|
|
1228
1360
|
}
|
|
1229
|
-
//
|
|
1230
|
-
//
|
|
1231
|
-
this.
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1361
|
+
// Reserve order before reading a Blob: later audio and controls must not
|
|
1362
|
+
// overtake it. Closing still cancels queued work rather than replaying it.
|
|
1363
|
+
this.queueOutbound(async () => omniAudioFrame(new Uint8Array(await (chunk as Blob).arrayBuffer())));
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
private queueOutbound(frame: () => ArrayBufferLike | ArrayBufferView | Promise<ArrayBufferLike | ArrayBufferView>): void {
|
|
1367
|
+
this.outboundQueued++;
|
|
1368
|
+
this.blobTail = this.blobTail.then(async () => {
|
|
1369
|
+
this.assertSendable();
|
|
1370
|
+
const data = await frame();
|
|
1371
|
+
this.assertSendable();
|
|
1372
|
+
this.ws.send(data);
|
|
1373
|
+
}).catch((error) => {
|
|
1374
|
+
this.opts.onError?.(error instanceof Error ? error : new Error(String(error)));
|
|
1375
|
+
}).finally(() => { this.outboundQueued--; });
|
|
1238
1376
|
}
|
|
1239
1377
|
|
|
1240
|
-
|
|
1378
|
+
private sendOrdered(data: ArrayBufferLike | ArrayBufferView): void {
|
|
1379
|
+
if (this.outboundQueued) this.queueOutbound(() => data);
|
|
1380
|
+
else this.ws.send(data);
|
|
1381
|
+
}
|
|
1382
|
+
|
|
1383
|
+
/** Report samples actually rendered by the player in one reply segment.
|
|
1384
|
+
* Receiving or scheduling bytes is not playback. Stale receipts are ignored
|
|
1385
|
+
* server-side; this never changes completed tool actions. */
|
|
1386
|
+
playbackProgress(replyId: string, segmentId: string, playedSamples: number): void {
|
|
1387
|
+
this.assertSendable();
|
|
1388
|
+
if (!this.playbackEnabled) throw new Error("Playback acknowledgments were not negotiated");
|
|
1389
|
+
if (!/^[a-f0-9]{32}$/.test(replyId) || !/^[1-9][0-9]{0,5}$/.test(segmentId)
|
|
1390
|
+
|| !Number.isSafeInteger(playedSamples) || playedSamples < 0) {
|
|
1391
|
+
throw new TypeError("Invalid playback progress");
|
|
1392
|
+
}
|
|
1393
|
+
this.sendOrdered(omniControlFrame({ type: "playback_progress", reply_id: replyId,
|
|
1394
|
+
segment_id: segmentId, played_samples: playedSamples }));
|
|
1395
|
+
}
|
|
1396
|
+
|
|
1397
|
+
/** Send a nonempty DTMF sequence as ordered single-key control frames.
|
|
1398
|
+
* Valid keys are 0–9, *, # and A–D; lowercase a–d are normalized. */
|
|
1241
1399
|
sendDtmf(digits: string): void {
|
|
1242
1400
|
this.assertSendable();
|
|
1243
|
-
|
|
1401
|
+
if (typeof digits !== "string" || digits.length === 0 || /[^0-9*#A-Da-d]/.test(digits)) {
|
|
1402
|
+
throw new TypeError("Invalid DTMF sequence");
|
|
1403
|
+
}
|
|
1404
|
+
for (const digit of digits.toUpperCase()) {
|
|
1405
|
+
this.sendOrdered(omniControlFrame({ type: "dtmf", digit }));
|
|
1406
|
+
}
|
|
1244
1407
|
}
|
|
1245
1408
|
|
|
1246
1409
|
/** Reply to a client-loop {@link OmniEvent.ToolCall}. */
|
|
1247
1410
|
toolResult(callId: string, payload: { result?: unknown; error?: string }): void {
|
|
1248
1411
|
this.assertSendable();
|
|
1249
|
-
this.
|
|
1412
|
+
this.sendOrdered(omniControlFrame({ type: "tool_result", call_id: callId, ...payload }));
|
|
1250
1413
|
}
|
|
1251
1414
|
|
|
1252
1415
|
/**
|
|
@@ -1261,7 +1424,7 @@ export class OmniConnection {
|
|
|
1261
1424
|
}
|
|
1262
1425
|
const payload = { ...frame };
|
|
1263
1426
|
delete payload.event;
|
|
1264
|
-
this.
|
|
1427
|
+
this.sendOrdered(omniControlFrame(payload));
|
|
1265
1428
|
}
|
|
1266
1429
|
|
|
1267
1430
|
/** Close the session. */
|
|
@@ -1955,6 +2118,7 @@ export class PyAI {
|
|
|
1955
2118
|
headers: this.authHeaders({ "Content-Type": "application/json" }),
|
|
1956
2119
|
body: JSON.stringify({ model: "pyai-speak", ...params }),
|
|
1957
2120
|
});
|
|
2121
|
+
await validateSpeechResponse(res, params);
|
|
1958
2122
|
return res.arrayBuffer();
|
|
1959
2123
|
},
|
|
1960
2124
|
/**
|
|
@@ -1972,6 +2136,7 @@ export class PyAI {
|
|
|
1972
2136
|
headers: this.authHeaders({ "Content-Type": "application/json" }),
|
|
1973
2137
|
body: JSON.stringify({ model: "pyai-speak", ...params, stream: true }),
|
|
1974
2138
|
});
|
|
2139
|
+
await validateSpeechResponse(res, params);
|
|
1975
2140
|
if (!res.body) throw new PyAIError(res.status, "Response had no body to stream");
|
|
1976
2141
|
return res.body as ReadableStream<Uint8Array>;
|
|
1977
2142
|
},
|
|
@@ -1982,7 +2147,7 @@ export class PyAI {
|
|
|
1982
2147
|
filename?: string;
|
|
1983
2148
|
model?: string;
|
|
1984
2149
|
/** Optional language hint. Omission enables automatic detection. */
|
|
1985
|
-
language?: "en" | "es" | "fr" | "de" | "hi" | "it" | "pt" | "nl";
|
|
2150
|
+
language?: "en" | "es" | "fr" | "de" | "hi" | "it" | "pt" | "nl" | "zh" | "yue" | "ar" | "id" | "ko" | "th" | "vi" | "ja" | "tr" | "ms" | "sv" | "da" | "fi" | "pl" | "cs" | "fil" | "fa" | "el" | "hu" | "mk" | "ro";
|
|
1986
2151
|
numerals?: boolean;
|
|
1987
2152
|
smart_format?: boolean;
|
|
1988
2153
|
dictation?: boolean;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/** Validate only explicitly requested PCM; keep opaque container APIs unchanged. */
|
|
2
|
+
export async function validateSpeechResponse(
|
|
3
|
+
response: Response,
|
|
4
|
+
params: { response_format?: string; sample_rate?: number },
|
|
5
|
+
): Promise<void> {
|
|
6
|
+
if (params.response_format !== "pcm") return;
|
|
7
|
+
const rate = params.sample_rate ?? 24000;
|
|
8
|
+
const parts = (response.headers.get("Content-Type") ?? "").split(";");
|
|
9
|
+
const mediaType = parts.shift()?.trim().toLowerCase();
|
|
10
|
+
const values = new Map<string, string[]>();
|
|
11
|
+
let malformed = false;
|
|
12
|
+
for (const part of parts) {
|
|
13
|
+
const match = /^\s*([^=\s]+)\s*=\s*(?:"([^"\\]*)"|([^\s";]+))\s*$/.exec(part);
|
|
14
|
+
if (!match) { malformed = true; continue; }
|
|
15
|
+
const name = match[1]!.toLowerCase();
|
|
16
|
+
values.set(name, [...(values.get(name) ?? []), match[2] ?? match[3]!]);
|
|
17
|
+
}
|
|
18
|
+
const rates = values.get("rate") ?? [];
|
|
19
|
+
const channels = values.get("channels") ?? [];
|
|
20
|
+
const expected = {
|
|
21
|
+
"X-PyAI-Audio-Encoding": "pcm_s16le",
|
|
22
|
+
"X-PyAI-Audio-Sample-Rate": String(rate),
|
|
23
|
+
"X-PyAI-Audio-Channels": "1",
|
|
24
|
+
};
|
|
25
|
+
if (malformed || !Number.isInteger(rate) || rate < 8000 || rate > 48000
|
|
26
|
+
|| mediaType !== "audio/pcm" || rates.length !== 1 || rates[0] !== String(rate)
|
|
27
|
+
|| (channels.length > 0 && (channels.length !== 1 || channels[0] !== "1"))
|
|
28
|
+
|| Object.entries(expected).some(([key, value]) => response.headers.has(key) && response.headers.get(key) !== value)) {
|
|
29
|
+
// A response body owns the fetch connection even before any audio is read.
|
|
30
|
+
try { await response.body?.cancel(); } catch { /* Preserve the metadata error. */ }
|
|
31
|
+
throw new Error("Speak PCM response metadata must match requested mono PCM16 sample rate");
|
|
32
|
+
}
|
|
33
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/** Explicit English speech formatting. Plain strings are never rewritten. */
|
|
2
|
+
export type SpeechTextPart = string
|
|
3
|
+
| { type: "currency"; value: string; currency: "USD" | "GBP" | "EUR" }
|
|
4
|
+
| { type: "date"; value: string; locale: "en-US" | "en-GB" }
|
|
5
|
+
| { type: "phone" | "digits"; value: string };
|
|
6
|
+
|
|
7
|
+
const small = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine",
|
|
8
|
+
"ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen"];
|
|
9
|
+
const tens = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"];
|
|
10
|
+
const ordinals = ["", "first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eighth", "ninth",
|
|
11
|
+
"tenth", "eleventh", "twelfth", "thirteenth", "fourteenth", "fifteenth", "sixteenth", "seventeenth", "eighteenth", "nineteenth",
|
|
12
|
+
"twentieth", "twenty first", "twenty second", "twenty third", "twenty fourth", "twenty fifth", "twenty sixth", "twenty seventh", "twenty eighth", "twenty ninth", "thirtieth", "thirty first"];
|
|
13
|
+
const months = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"];
|
|
14
|
+
|
|
15
|
+
function words(n: number): string {
|
|
16
|
+
if (n < 20) return small[n]!;
|
|
17
|
+
if (n < 100) return tens[Math.floor(n / 10)]! + (n % 10 ? " " + words(n % 10) : "");
|
|
18
|
+
for (const [scale, label] of [[1_000_000, "million"], [1_000, "thousand"], [100, "hundred"]] as const) {
|
|
19
|
+
if (n >= scale) return words(Math.floor(n / scale)) + " " + label + (n % scale ? " " + words(n % scale) : "");
|
|
20
|
+
}
|
|
21
|
+
throw new RangeError("Unsupported number");
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function formatPart(part: SpeechTextPart): string {
|
|
25
|
+
if (typeof part === "string") return part;
|
|
26
|
+
if (!part || typeof part.value !== "string") throw new TypeError("Speech fields require a string value");
|
|
27
|
+
const value = part.value;
|
|
28
|
+
switch (part.type) {
|
|
29
|
+
case "digits":
|
|
30
|
+
if (!/^[0-9]{1,64}$/.test(value)) throw new RangeError("digits requires 1–64 decimal digits");
|
|
31
|
+
return [...value].map(d => small[Number(d)]!).join(" ");
|
|
32
|
+
case "phone": {
|
|
33
|
+
// Formatting separators are allowed; extensions and country codes are not inferred.
|
|
34
|
+
if (!/^\+?[0-9 ().-]+$/.test(value) || value.length > 64) throw new RangeError("Unsupported phone format");
|
|
35
|
+
const digits = value.replace(/[^0-9]/g, "");
|
|
36
|
+
if (digits.length < 3 || digits.length > 15) throw new RangeError("phone requires 3–15 digits");
|
|
37
|
+
return (value.startsWith("+") ? "plus " : "") + [...digits].map(d => small[Number(d)]!).join(" ");
|
|
38
|
+
}
|
|
39
|
+
case "currency": {
|
|
40
|
+
// Decimal strings avoid binary floating point rounding and ambiguous separators.
|
|
41
|
+
if (!/^-?(0|[1-9][0-9]{0,8})(\.[0-9]{1,2})?$/.test(value)) throw new RangeError("currency requires a decimal string with at most two fractional digits");
|
|
42
|
+
const units = { USD: ["US dollar", "US dollars", "cent", "cents"], GBP: ["pound", "pounds", "penny", "pence"], EUR: ["euro", "euros", "cent", "cents"] } as const;
|
|
43
|
+
if (!Object.hasOwn(units, part.currency)) throw new RangeError("Unsupported currency");
|
|
44
|
+
const [wholeText, fraction = ""] = value.replace(/^-/, "").split(".");
|
|
45
|
+
const whole = Number(wholeText), minor = Number(fraction.padEnd(2, "0"));
|
|
46
|
+
const names = units[part.currency];
|
|
47
|
+
return (value.startsWith("-") && (whole || minor) ? "minus " : "")
|
|
48
|
+
+ words(whole) + " " + names[whole === 1 ? 0 : 1]
|
|
49
|
+
+ (minor ? " and " + words(minor) + " " + names[minor === 1 ? 2 : 3] : "");
|
|
50
|
+
}
|
|
51
|
+
case "date": {
|
|
52
|
+
if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) throw new RangeError("date requires YYYY-MM-DD");
|
|
53
|
+
if (part.locale !== "en-US" && part.locale !== "en-GB") throw new RangeError("Unsupported date locale");
|
|
54
|
+
const [year, month, day] = value.split("-").map(Number) as [number, number, number];
|
|
55
|
+
const leap = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0);
|
|
56
|
+
const limit = [31, leap ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31][month - 1];
|
|
57
|
+
if (year < 1 || !limit || day < 1 || day > limit) throw new RangeError("Invalid calendar date");
|
|
58
|
+
return (part.locale === "en-US" ? months[month - 1] + " " + ordinals[day]
|
|
59
|
+
: "the " + ordinals[day] + " of " + months[month - 1]) + ", " + words(year);
|
|
60
|
+
}
|
|
61
|
+
default: throw new TypeError("Unsupported speech field type");
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Join literal text and explicitly typed English fields for Speak input or Cast text.
|
|
66
|
+
* Supply spaces and punctuation in literal parts. This does not infer field types,
|
|
67
|
+
* validate phone reachability, or guarantee how a voice will pronounce the output.
|
|
68
|
+
*/
|
|
69
|
+
export function formatSpeechText(parts: readonly SpeechTextPart[]): string {
|
|
70
|
+
return parts.map(formatPart).join("");
|
|
71
|
+
}
|