@fabric-harness/sdk 1.10.0 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +14 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -0
- package/dist/index.js.map +1 -1
- package/dist/model-pricing.d.ts +6 -0
- package/dist/model-pricing.d.ts.map +1 -1
- package/dist/model-pricing.js +17 -0
- package/dist/model-pricing.js.map +1 -1
- package/dist/model.d.ts +4 -0
- package/dist/model.d.ts.map +1 -1
- package/dist/model.js.map +1 -1
- package/dist/strict.d.ts +14 -0
- package/dist/strict.d.ts.map +1 -1
- package/dist/strict.js +6 -0
- package/dist/strict.js.map +1 -1
- package/dist/voice-openai-realtime.d.ts.map +1 -1
- package/dist/voice-openai-realtime.js +32 -0
- package/dist/voice-openai-realtime.js.map +1 -1
- package/dist/voice-pipeline.d.ts +49 -0
- package/dist/voice-pipeline.d.ts.map +1 -0
- package/dist/voice-pipeline.js +312 -0
- package/dist/voice-pipeline.js.map +1 -0
- package/dist/voice-stt-cartesia.d.ts +29 -0
- package/dist/voice-stt-cartesia.d.ts.map +1 -0
- package/dist/voice-stt-cartesia.js +160 -0
- package/dist/voice-stt-cartesia.js.map +1 -0
- package/dist/voice-stt-deepgram.d.ts +28 -0
- package/dist/voice-stt-deepgram.d.ts.map +1 -0
- package/dist/voice-stt-deepgram.js +174 -0
- package/dist/voice-stt-deepgram.js.map +1 -0
- package/dist/voice-stt.d.ts +72 -0
- package/dist/voice-stt.d.ts.map +1 -0
- package/dist/voice-stt.js +8 -0
- package/dist/voice-stt.js.map +1 -0
- package/dist/voice-tts-cartesia.d.ts +36 -0
- package/dist/voice-tts-cartesia.d.ts.map +1 -0
- package/dist/voice-tts-cartesia.js +159 -0
- package/dist/voice-tts-cartesia.js.map +1 -0
- package/dist/voice-tts-elevenlabs.d.ts +35 -0
- package/dist/voice-tts-elevenlabs.d.ts.map +1 -0
- package/dist/voice-tts-elevenlabs.js +163 -0
- package/dist/voice-tts-elevenlabs.js.map +1 -0
- package/dist/voice-tts.d.ts +50 -0
- package/dist/voice-tts.d.ts.map +1 -0
- package/dist/voice-tts.js +11 -0
- package/dist/voice-tts.js.map +1 -0
- package/dist/voice-ws-client.d.ts +77 -0
- package/dist/voice-ws-client.d.ts.map +1 -0
- package/dist/voice-ws-client.js +115 -0
- package/dist/voice-ws-client.js.map +1 -0
- package/dist/voice-ws-shared.d.ts +20 -0
- package/dist/voice-ws-shared.d.ts.map +1 -0
- package/dist/voice-ws-shared.js +42 -0
- package/dist/voice-ws-shared.js.map +1 -0
- package/dist/voice.d.ts +18 -0
- package/dist/voice.d.ts.map +1 -1
- package/package.json +8 -8
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cartesia streaming TTS provider.
|
|
3
|
+
*
|
|
4
|
+
* Uses the Cartesia WS endpoint (`wss://api.cartesia.ai/tts/websocket`).
|
|
5
|
+
* `sonic-2` is the v1.12 default — Cartesia's flagship low-latency model.
|
|
6
|
+
*
|
|
7
|
+
* Pricing: per character. Rates registered in `model-pricing.ts` under
|
|
8
|
+
* provider `cartesia`.
|
|
9
|
+
*/
|
|
10
|
+
import { base64ToBytes, openVoiceWs, toError } from './voice-ws-shared.js';
|
|
11
|
+
const DEFAULT_BASE = 'wss://api.cartesia.ai';
|
|
12
|
+
const DEFAULT_MODEL = 'sonic-2';
|
|
13
|
+
const DEFAULT_API_VERSION = '2024-11-13';
|
|
14
|
+
// Cartesia 'Newsman' voice — a public default. Override per-call.
|
|
15
|
+
const DEFAULT_VOICE = 'd46abd1d-2d02-43e8-819f-51fb652c1c61';
|
|
16
|
+
export class CartesiaTtsProvider {
|
|
17
|
+
name;
|
|
18
|
+
apiKey;
|
|
19
|
+
defaultVoice;
|
|
20
|
+
model;
|
|
21
|
+
apiVersion;
|
|
22
|
+
baseUrl;
|
|
23
|
+
last;
|
|
24
|
+
constructor(options) {
|
|
25
|
+
if (!options.apiKey)
|
|
26
|
+
throw new Error('CartesiaTtsProvider: apiKey is required.');
|
|
27
|
+
this.apiKey = options.apiKey;
|
|
28
|
+
this.defaultVoice = options.defaultVoice ?? DEFAULT_VOICE;
|
|
29
|
+
this.model = options.model ?? DEFAULT_MODEL;
|
|
30
|
+
this.apiVersion = options.apiVersion ?? DEFAULT_API_VERSION;
|
|
31
|
+
this.baseUrl = options.baseUrl ?? DEFAULT_BASE;
|
|
32
|
+
this.name = options.name ?? 'cartesia';
|
|
33
|
+
}
|
|
34
|
+
lastUsage() {
|
|
35
|
+
return this.last;
|
|
36
|
+
}
|
|
37
|
+
synthesize(text, opts = {}) {
|
|
38
|
+
const voice = opts.voice ?? this.defaultVoice;
|
|
39
|
+
const format = opts.audioFormat ?? 'pcm16';
|
|
40
|
+
const sampleRate = opts.sampleRate ?? defaultSampleRate(format);
|
|
41
|
+
const startedAt = Date.now();
|
|
42
|
+
let totalBytes = 0;
|
|
43
|
+
const charCount = text.length;
|
|
44
|
+
const provider = this;
|
|
45
|
+
return {
|
|
46
|
+
[Symbol.asyncIterator]: async function* () {
|
|
47
|
+
const url = `${provider.baseUrl}/tts/websocket?cartesia_version=${encodeURIComponent(provider.apiVersion)}&api_key=${encodeURIComponent(provider.apiKey)}`;
|
|
48
|
+
const ws = await openVoiceWs(url, {});
|
|
49
|
+
const queue = [];
|
|
50
|
+
const waiters = [];
|
|
51
|
+
let closed = false;
|
|
52
|
+
let errorMsg;
|
|
53
|
+
const push = (chunk) => {
|
|
54
|
+
if (closed)
|
|
55
|
+
return;
|
|
56
|
+
if (!chunk) {
|
|
57
|
+
closed = true;
|
|
58
|
+
while (waiters.length > 0)
|
|
59
|
+
waiters.shift()({ value: undefined, done: true });
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
totalBytes += chunk.byteLength;
|
|
63
|
+
const w = waiters.shift();
|
|
64
|
+
if (w)
|
|
65
|
+
w({ value: chunk, done: false });
|
|
66
|
+
else
|
|
67
|
+
queue.push(chunk);
|
|
68
|
+
};
|
|
69
|
+
ws.addEventListener('message', (ev) => {
|
|
70
|
+
const raw = typeof ev.data === 'string' ? ev.data : ev.data?.toString?.() ?? '';
|
|
71
|
+
let payload;
|
|
72
|
+
try {
|
|
73
|
+
payload = JSON.parse(raw);
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
const type = String(payload.type ?? '');
|
|
79
|
+
if (type === 'chunk' && typeof payload.data === 'string') {
|
|
80
|
+
push(base64ToBytes(payload.data));
|
|
81
|
+
}
|
|
82
|
+
else if (type === 'done') {
|
|
83
|
+
push(undefined);
|
|
84
|
+
try {
|
|
85
|
+
ws.close();
|
|
86
|
+
}
|
|
87
|
+
catch { }
|
|
88
|
+
}
|
|
89
|
+
else if (type === 'error') {
|
|
90
|
+
errorMsg = String(payload.error ?? payload.message ?? 'cartesia error');
|
|
91
|
+
push(undefined);
|
|
92
|
+
try {
|
|
93
|
+
ws.close();
|
|
94
|
+
}
|
|
95
|
+
catch { }
|
|
96
|
+
}
|
|
97
|
+
});
|
|
98
|
+
ws.addEventListener('error', (ev) => {
|
|
99
|
+
errorMsg = toError(ev, 'Cartesia WS error').message;
|
|
100
|
+
push(undefined);
|
|
101
|
+
});
|
|
102
|
+
ws.addEventListener('close', () => push(undefined));
|
|
103
|
+
const outputContainer = format === 'g711_ulaw' ? 'mulaw' : 'raw';
|
|
104
|
+
const encoding = format === 'g711_ulaw' ? 'pcm_mulaw' : 'pcm_s16le';
|
|
105
|
+
ws.send(JSON.stringify({
|
|
106
|
+
model_id: provider.model,
|
|
107
|
+
voice: { mode: 'id', id: voice },
|
|
108
|
+
transcript: text,
|
|
109
|
+
continue: false,
|
|
110
|
+
context_id: cryptoRandomId(),
|
|
111
|
+
language: 'en',
|
|
112
|
+
output_format: {
|
|
113
|
+
container: outputContainer,
|
|
114
|
+
encoding,
|
|
115
|
+
sample_rate: sampleRate,
|
|
116
|
+
},
|
|
117
|
+
...(opts.speed !== undefined ? { speed: opts.speed } : {}),
|
|
118
|
+
}));
|
|
119
|
+
opts.signal?.addEventListener('abort', () => { try {
|
|
120
|
+
ws.close();
|
|
121
|
+
}
|
|
122
|
+
catch { } push(undefined); }, { once: true });
|
|
123
|
+
try {
|
|
124
|
+
while (true) {
|
|
125
|
+
if (queue.length > 0) {
|
|
126
|
+
yield queue.shift();
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
if (closed)
|
|
130
|
+
break;
|
|
131
|
+
const next = await new Promise((resolve) => waiters.push(resolve));
|
|
132
|
+
if (next.done)
|
|
133
|
+
break;
|
|
134
|
+
yield next.value;
|
|
135
|
+
}
|
|
136
|
+
if (errorMsg)
|
|
137
|
+
throw new Error(`Cartesia TTS: ${errorMsg}`);
|
|
138
|
+
}
|
|
139
|
+
finally {
|
|
140
|
+
provider.last = {
|
|
141
|
+
characters: charCount,
|
|
142
|
+
audioBytes: totalBytes,
|
|
143
|
+
durationMs: Date.now() - startedAt,
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
},
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
function defaultSampleRate(format) {
|
|
151
|
+
if (format === 'g711_ulaw' || format === 'g711_alaw')
|
|
152
|
+
return 8000;
|
|
153
|
+
return 24000;
|
|
154
|
+
}
|
|
155
|
+
function cryptoRandomId() {
|
|
156
|
+
// Lightweight per-call id — Cartesia accepts any unique string.
|
|
157
|
+
return `fh-${Math.random().toString(36).slice(2, 10)}-${Date.now().toString(36)}`;
|
|
158
|
+
}
|
|
159
|
+
//# sourceMappingURL=voice-tts-cartesia.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voice-tts-cartesia.js","sourceRoot":"","sources":["../src/voice-tts-cartesia.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,OAAO,EAAuB,MAAM,sBAAsB,CAAC;AAkBhG,MAAM,YAAY,GAAG,uBAAuB,CAAC;AAC7C,MAAM,aAAa,GAAG,SAAS,CAAC;AAChC,MAAM,mBAAmB,GAAG,YAAY,CAAC;AACzC,kEAAkE;AAClE,MAAM,aAAa,GAAG,sCAAsC,CAAC;AAE7D,MAAM,OAAO,mBAAmB;IACrB,IAAI,CAAS;IACL,MAAM,CAAS;IACf,YAAY,CAAS;IACrB,KAAK,CAAS;IACd,UAAU,CAAS;IACnB,OAAO,CAAS;IACzB,IAAI,CAAqB;IAEjC,YAAY,OAAmC;QAC7C,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QACjF,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC;QAC1D,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,aAAa,CAAC;QAC5C,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,mBAAmB,CAAC;QAC5D,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,YAAY,CAAC;QAC/C,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,UAAU,CAAC;IACzC,CAAC;IAED,SAAS;QACP,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,UAAU,CAAC,IAAY,EAAE,OAA4B,EAAE;QACrD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,YAAY,CAAC;QAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,IAAI,OAAO,CAAC;QAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAChE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC;QAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC;QAEtB,OAAO;YACL,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,KAAK,SAAS,CAAC;gBACrC,MAAM,GAAG,GAAG,GAAG,QAAQ,CAAC,OAAO,mCAAmC,kBAAkB,CAAC,QAAQ,CAAC,UAAU,CAAC,YAAY,kBAAkB,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3J,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBACtC,MAAM,KAAK,GAAiB,EAAE,CAAC;gBAC/B,MAAM,OAAO,GAAuD,EAAE,CAAC;gBACvE,IAAI,MAAM,GAAG,KAAK,CAAC;gBACnB,IAAI,QAA4B,CAAC;gBAEjC,MAAM,IAAI,GAAG,CAAC,KAA6B,EAAE,EAAE;oBAC7C,IAAI,MAAM;wBAAE,OAAO;oBACnB,IAAI,CAAC,KAAK,EAAE,CAAC;wBACX,MAAM,GAAG,IAAI,CAAC;wBACd,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC;4BAAE,OAAO,CAAC,KAAK,EAAG,CAAC,EAAE,KAAK,EAAE,SAAkC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;wBACvG,OAAO;oBACT,CAAC;oBACD,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;oBAC/B,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;oBAC1B,IAAI,CAAC;wBAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;;wBACnC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACzB,CAAC,CAAC;gBAEF,EAAE,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE;oBACpC,MAAM,GAAG,GAAG,OAAO,EAAE,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAE,EAAE,CAAC,IAAoC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC;oBACjH,IAAI,OAAgC,CAAC;oBACrC,IAAI,CAAC;wBAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC;oBAAC,CAAC;oBAAC,MAAM,CAAC;wBAAC,OAAO;oBAAC,CAAC;oBAC/E,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;oBACxC,IAAI,IAAI,KAAK,OAAO,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBACzD,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;oBACpC,CAAC;yBAAM,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;wBAC3B,IAAI,CAAC,SAAS,CAAC,CAAC;wBAChB,IAAI,CAAC;4BAAC,EAAE,CAAC,KAAK,EAAE,CAAC;wBAAC,CAAC;wBAAC,MAAM,CAAC,CAAA,CAAC;oBAC9B,CAAC;yBAAM,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;wBAC5B,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,OAAO,IAAI,gBAAgB,CAAC,CAAC;wBACxE,IAAI,CAAC,SAAS,CAAC,CAAC;wBAChB,IAAI,CAAC;4BAAC,EAAE,CAAC,KAAK,EAAE,CAAC;wBAAC,CAAC;wBAAC,MAAM,CAAC,CAAA,CAAC;oBAC9B,CAAC;gBACH,CAAC,CAAC,CAAC;gBACH,EAAE,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,EAAE;oBAClC,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,mBAAmB,CAAC,CAAC,OAAO,CAAC;oBACpD,IAAI,CAAC,SAAS,CAAC,CAAC;gBAClB,CAAC,CAAC,CAAC;gBACH,EAAE,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;gBAEpD,MAAM,eAAe,GAAG,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;gBACjE,MAAM,QAAQ,GAAG,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC;gBAEpE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;oBACrB,QAAQ,EAAE,QAAQ,CAAC,KAAK;oBACxB,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE;oBAChC,UAAU,EAAE,IAAI;oBAChB,QAAQ,EAAE,KAAK;oBACf,UAAU,EAAE,cAAc,EAAE;oBAC5B,QAAQ,EAAE,IAAI;oBACd,aAAa,EAAE;wBACb,SAAS,EAAE,eAAe;wBAC1B,QAAQ;wBACR,WAAW,EAAE,UAAU;qBACxB;oBACD,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC3D,CAAC,CAAC,CAAC;gBAEJ,IAAI,CAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;oBAAC,EAAE,CAAC,KAAK,EAAE,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAA,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBAEhH,IAAI,CAAC;oBACH,OAAO,IAAI,EAAE,CAAC;wBACZ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACrB,MAAM,KAAK,CAAC,KAAK,EAAG,CAAC;4BACrB,SAAS;wBACX,CAAC;wBACD,IAAI,MAAM;4BAAE,MAAM;wBAClB,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAA6B,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;wBAC/F,IAAI,IAAI,CAAC,IAAI;4BAAE,MAAM;wBACrB,MAAM,IAAI,CAAC,KAAK,CAAC;oBACnB,CAAC;oBACD,IAAI,QAAQ;wBAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,QAAQ,EAAE,CAAC,CAAC;gBAC7D,CAAC;wBAAS,CAAC;oBACT,QAAQ,CAAC,IAAI,GAAG;wBACd,UAAU,EAAE,SAAS;wBACrB,UAAU,EAAE,UAAU;wBACtB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;qBACnC,CAAC;gBACJ,CAAC;YACH,CAAC;SACF,CAAC;IACJ,CAAC;CACF;AAED,SAAS,iBAAiB,CAAC,MAAwB;IACjD,IAAI,MAAM,KAAK,WAAW,IAAI,MAAM,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IAClE,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc;IACrB,gEAAgE;IAChE,OAAO,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;AACpF,CAAC"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ElevenLabs streaming TTS provider.
|
|
3
|
+
*
|
|
4
|
+
* Uses the WebSocket streaming endpoint
|
|
5
|
+
* (`wss://api.elevenlabs.io/v1/text-to-speech/:voiceId/stream-input`) so
|
|
6
|
+
* audio frames stream out as text arrives. PCM 16-bit little-endian at
|
|
7
|
+
* configurable sample rate; μ-law 8kHz available for telephony.
|
|
8
|
+
*
|
|
9
|
+
* Pricing: billed per character of input text. Rates registered in
|
|
10
|
+
* `model-pricing.ts` under provider `elevenlabs`.
|
|
11
|
+
*/
|
|
12
|
+
import type { TtsProvider, TtsSynthesisOptions, TtsSynthesisUsage } from './voice-tts.js';
|
|
13
|
+
export interface ElevenLabsTtsProviderOptions {
|
|
14
|
+
apiKey: string;
|
|
15
|
+
/** Default voice id when `synthesize` doesn't override. */
|
|
16
|
+
defaultVoice?: string;
|
|
17
|
+
/** Default model id. `eleven_turbo_v2_5` is the v1.12 default (low latency). */
|
|
18
|
+
model?: string;
|
|
19
|
+
/** Override base URL — useful for region-pinned deployments. */
|
|
20
|
+
baseUrl?: string;
|
|
21
|
+
/** Provider name override for telemetry. Default `elevenlabs`. */
|
|
22
|
+
name?: string;
|
|
23
|
+
}
|
|
24
|
+
export declare class ElevenLabsTtsProvider implements TtsProvider {
|
|
25
|
+
readonly name: string;
|
|
26
|
+
private readonly apiKey;
|
|
27
|
+
private readonly defaultVoice;
|
|
28
|
+
private readonly model;
|
|
29
|
+
private readonly baseUrl;
|
|
30
|
+
private last?;
|
|
31
|
+
constructor(options: ElevenLabsTtsProviderOptions);
|
|
32
|
+
lastUsage(): TtsSynthesisUsage | undefined;
|
|
33
|
+
synthesize(text: string, opts?: TtsSynthesisOptions): AsyncIterable<Uint8Array>;
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=voice-tts-elevenlabs.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voice-tts-elevenlabs.d.ts","sourceRoot":"","sources":["../src/voice-tts-elevenlabs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAG1F,MAAM,WAAW,4BAA4B;IAC3C,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gFAAgF;IAChF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gEAAgE;IAChE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kEAAkE;IAClE,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAMD,qBAAa,qBAAsB,YAAW,WAAW;IACvD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,IAAI,CAAC,CAAoB;gBAErB,OAAO,EAAE,4BAA4B;IASjD,SAAS,IAAI,iBAAiB,GAAG,SAAS;IAI1C,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,mBAAwB,GAAG,aAAa,CAAC,UAAU,CAAC;CAwFpF"}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ElevenLabs streaming TTS provider.
|
|
3
|
+
*
|
|
4
|
+
* Uses the WebSocket streaming endpoint
|
|
5
|
+
* (`wss://api.elevenlabs.io/v1/text-to-speech/:voiceId/stream-input`) so
|
|
6
|
+
* audio frames stream out as text arrives. PCM 16-bit little-endian at
|
|
7
|
+
* configurable sample rate; μ-law 8kHz available for telephony.
|
|
8
|
+
*
|
|
9
|
+
* Pricing: billed per character of input text. Rates registered in
|
|
10
|
+
* `model-pricing.ts` under provider `elevenlabs`.
|
|
11
|
+
*/
|
|
12
|
+
import { base64ToBytes, openVoiceWs, toError } from './voice-ws-shared.js';
|
|
13
|
+
const DEFAULT_BASE = 'wss://api.elevenlabs.io';
|
|
14
|
+
const DEFAULT_MODEL = 'eleven_turbo_v2_5';
|
|
15
|
+
const DEFAULT_VOICE = '21m00Tcm4TlvDq8ikWAM'; // 'Rachel' — ElevenLabs starter voice.
|
|
16
|
+
export class ElevenLabsTtsProvider {
|
|
17
|
+
name;
|
|
18
|
+
apiKey;
|
|
19
|
+
defaultVoice;
|
|
20
|
+
model;
|
|
21
|
+
baseUrl;
|
|
22
|
+
last;
|
|
23
|
+
constructor(options) {
|
|
24
|
+
if (!options.apiKey)
|
|
25
|
+
throw new Error('ElevenLabsTtsProvider: apiKey is required.');
|
|
26
|
+
this.apiKey = options.apiKey;
|
|
27
|
+
this.defaultVoice = options.defaultVoice ?? DEFAULT_VOICE;
|
|
28
|
+
this.model = options.model ?? DEFAULT_MODEL;
|
|
29
|
+
this.baseUrl = options.baseUrl ?? DEFAULT_BASE;
|
|
30
|
+
this.name = options.name ?? 'elevenlabs';
|
|
31
|
+
}
|
|
32
|
+
lastUsage() {
|
|
33
|
+
return this.last;
|
|
34
|
+
}
|
|
35
|
+
synthesize(text, opts = {}) {
|
|
36
|
+
const voice = opts.voice ?? this.defaultVoice;
|
|
37
|
+
const format = opts.audioFormat ?? 'pcm16';
|
|
38
|
+
const sampleRate = opts.sampleRate ?? defaultSampleRate(format);
|
|
39
|
+
const startedAt = Date.now();
|
|
40
|
+
let totalBytes = 0;
|
|
41
|
+
const charCount = text.length;
|
|
42
|
+
const provider = this;
|
|
43
|
+
return {
|
|
44
|
+
[Symbol.asyncIterator]: async function* () {
|
|
45
|
+
const url = buildUrl(provider.baseUrl, voice, provider.model, format, sampleRate);
|
|
46
|
+
const ws = await openVoiceWs(url, { 'xi-api-key': provider.apiKey });
|
|
47
|
+
const queue = [];
|
|
48
|
+
const waiters = [];
|
|
49
|
+
let closed = false;
|
|
50
|
+
let errorMsg;
|
|
51
|
+
const push = (chunk) => {
|
|
52
|
+
if (closed)
|
|
53
|
+
return;
|
|
54
|
+
if (!chunk) {
|
|
55
|
+
closed = true;
|
|
56
|
+
while (waiters.length > 0)
|
|
57
|
+
waiters.shift()({ value: undefined, done: true });
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
totalBytes += chunk.byteLength;
|
|
61
|
+
const w = waiters.shift();
|
|
62
|
+
if (w)
|
|
63
|
+
w({ value: chunk, done: false });
|
|
64
|
+
else
|
|
65
|
+
queue.push(chunk);
|
|
66
|
+
};
|
|
67
|
+
ws.addEventListener('message', (ev) => {
|
|
68
|
+
const raw = typeof ev.data === 'string' ? ev.data : ev.data?.toString?.() ?? '';
|
|
69
|
+
let payload;
|
|
70
|
+
try {
|
|
71
|
+
payload = JSON.parse(raw);
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
if (typeof payload.audio === 'string' && payload.audio.length > 0) {
|
|
77
|
+
push(base64ToBytes(payload.audio));
|
|
78
|
+
}
|
|
79
|
+
if (payload.isFinal === true) {
|
|
80
|
+
push(undefined);
|
|
81
|
+
try {
|
|
82
|
+
ws.close();
|
|
83
|
+
}
|
|
84
|
+
catch { }
|
|
85
|
+
}
|
|
86
|
+
if (payload.error || payload.message) {
|
|
87
|
+
errorMsg = String(payload.error ?? payload.message);
|
|
88
|
+
push(undefined);
|
|
89
|
+
try {
|
|
90
|
+
ws.close();
|
|
91
|
+
}
|
|
92
|
+
catch { }
|
|
93
|
+
}
|
|
94
|
+
});
|
|
95
|
+
ws.addEventListener('error', (ev) => {
|
|
96
|
+
errorMsg = toError(ev, 'ElevenLabs WS error').message;
|
|
97
|
+
push(undefined);
|
|
98
|
+
});
|
|
99
|
+
ws.addEventListener('close', () => push(undefined));
|
|
100
|
+
// Per ElevenLabs streaming protocol: send BOS with voice settings,
|
|
101
|
+
// then text chunk(s), then EOS (empty text).
|
|
102
|
+
ws.send(JSON.stringify({
|
|
103
|
+
text: ' ',
|
|
104
|
+
voice_settings: { stability: 0.5, similarity_boost: 0.8, ...(opts.speed !== undefined ? { speed: opts.speed } : {}) },
|
|
105
|
+
xi_api_key: provider.apiKey,
|
|
106
|
+
}));
|
|
107
|
+
ws.send(JSON.stringify({ text: text + ' ', try_trigger_generation: true }));
|
|
108
|
+
ws.send(JSON.stringify({ text: '' }));
|
|
109
|
+
opts.signal?.addEventListener('abort', () => { try {
|
|
110
|
+
ws.close();
|
|
111
|
+
}
|
|
112
|
+
catch { } push(undefined); }, { once: true });
|
|
113
|
+
try {
|
|
114
|
+
while (true) {
|
|
115
|
+
if (queue.length > 0) {
|
|
116
|
+
yield queue.shift();
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
if (closed)
|
|
120
|
+
break;
|
|
121
|
+
const next = await new Promise((resolve) => waiters.push(resolve));
|
|
122
|
+
if (next.done)
|
|
123
|
+
break;
|
|
124
|
+
yield next.value;
|
|
125
|
+
}
|
|
126
|
+
if (errorMsg)
|
|
127
|
+
throw new Error(`ElevenLabs TTS: ${errorMsg}`);
|
|
128
|
+
}
|
|
129
|
+
finally {
|
|
130
|
+
provider.last = {
|
|
131
|
+
characters: charCount,
|
|
132
|
+
audioBytes: totalBytes,
|
|
133
|
+
durationMs: Date.now() - startedAt,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
},
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
function buildUrl(base, voiceId, model, format, sampleRate) {
|
|
141
|
+
const outputFormat = formatToElevenLabs(format, sampleRate);
|
|
142
|
+
const params = new URLSearchParams({
|
|
143
|
+
model_id: model,
|
|
144
|
+
output_format: outputFormat,
|
|
145
|
+
});
|
|
146
|
+
return `${base}/v1/text-to-speech/${encodeURIComponent(voiceId)}/stream-input?${params.toString()}`;
|
|
147
|
+
}
|
|
148
|
+
function formatToElevenLabs(format, sampleRate) {
|
|
149
|
+
if (format === 'g711_ulaw')
|
|
150
|
+
return 'ulaw_8000';
|
|
151
|
+
if (format === 'g711_alaw')
|
|
152
|
+
return 'ulaw_8000'; // ElevenLabs ships ulaw only.
|
|
153
|
+
// pcm16 family — sample rate selectable: 8000/16000/22050/24000/44100.
|
|
154
|
+
const allowed = [8000, 16000, 22050, 24000, 44100];
|
|
155
|
+
const rate = allowed.includes(sampleRate) ? sampleRate : 24000;
|
|
156
|
+
return `pcm_${rate}`;
|
|
157
|
+
}
|
|
158
|
+
function defaultSampleRate(format) {
|
|
159
|
+
if (format === 'g711_ulaw' || format === 'g711_alaw')
|
|
160
|
+
return 8000;
|
|
161
|
+
return 24000;
|
|
162
|
+
}
|
|
163
|
+
//# sourceMappingURL=voice-tts-elevenlabs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voice-tts-elevenlabs.js","sourceRoot":"","sources":["../src/voice-tts-elevenlabs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,OAAO,EAAuB,MAAM,sBAAsB,CAAC;AAgBhG,MAAM,YAAY,GAAG,yBAAyB,CAAC;AAC/C,MAAM,aAAa,GAAG,mBAAmB,CAAC;AAC1C,MAAM,aAAa,GAAG,sBAAsB,CAAC,CAAC,uCAAuC;AAErF,MAAM,OAAO,qBAAqB;IACvB,IAAI,CAAS;IACL,MAAM,CAAS;IACf,YAAY,CAAS;IACrB,KAAK,CAAS;IACd,OAAO,CAAS;IACzB,IAAI,CAAqB;IAEjC,YAAY,OAAqC;QAC/C,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QACnF,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC;QAC1D,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,aAAa,CAAC;QAC5C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,YAAY,CAAC;QAC/C,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,YAAY,CAAC;IAC3C,CAAC;IAED,SAAS;QACP,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,UAAU,CAAC,IAAY,EAAE,OAA4B,EAAE;QACrD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,YAAY,CAAC;QAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,IAAI,OAAO,CAAC;QAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAChE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC;QAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC;QAEtB,OAAO;YACL,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,KAAK,SAAS,CAAC;gBACrC,MAAM,GAAG,GAAG,QAAQ,CAAC,QAAQ,CAAC,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;gBAClF,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;gBACrE,MAAM,KAAK,GAAiB,EAAE,CAAC;gBAC/B,MAAM,OAAO,GAAuD,EAAE,CAAC;gBACvE,IAAI,MAAM,GAAG,KAAK,CAAC;gBACnB,IAAI,QAA4B,CAAC;gBAEjC,MAAM,IAAI,GAAG,CAAC,KAA6B,EAAE,EAAE;oBAC7C,IAAI,MAAM;wBAAE,OAAO;oBACnB,IAAI,CAAC,KAAK,EAAE,CAAC;wBACX,MAAM,GAAG,IAAI,CAAC;wBACd,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC;4BAAE,OAAO,CAAC,KAAK,EAAG,CAAC,EAAE,KAAK,EAAE,SAAkC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;wBACvG,OAAO;oBACT,CAAC;oBACD,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;oBAC/B,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;oBAC1B,IAAI,CAAC;wBAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;;wBACnC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACzB,CAAC,CAAC;gBAEF,EAAE,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE;oBACpC,MAAM,GAAG,GAAG,OAAO,EAAE,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAE,EAAE,CAAC,IAAoC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC;oBACjH,IAAI,OAAgC,CAAC;oBACrC,IAAI,CAAC;wBAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC;oBAAC,CAAC;oBAAC,MAAM,CAAC;wBAAC,OAAO;oBAAC,CAAC;oBAC/E,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAClE,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;oBACrC,CAAC;oBACD,IAAI,OAAO,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;wBAC7B,IAAI,CAAC,SAAS,CAAC,CAAC;wBAChB,IAAI,CAAC;4BAAC,EAAE,CAAC,KAAK,EAAE,CAAC;wBAAC,CAAC;wBAAC,MAAM,CAAC,CAAA,CAAC;oBAC9B,CAAC;oBACD,IAAI,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;wBACrC,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;wBACpD,IAAI,CAAC,SAAS,CAAC,CAAC;wBAChB,IAAI,CAAC;4BAAC,EAAE,CAAC,KAAK,EAAE,CAAC;wBAAC,CAAC;wBAAC,MAAM,CAAC,CAAA,CAAC;oBAC9B,CAAC;gBACH,CAAC,CAAC,CAAC;gBACH,EAAE,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,EAAE;oBAClC,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,qBAAqB,CAAC,CAAC,OAAO,CAAC;oBACtD,IAAI,CAAC,SAAS,CAAC,CAAC;gBAClB,CAAC,CAAC,CAAC;gBACH,EAAE,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;gBAEpD,mEAAmE;gBACnE,6CAA6C;gBAC7C,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;oBACrB,IAAI,EAAE,GAAG;oBACT,cAAc,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,gBAAgB,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE;oBACrH,UAAU,EAAE,QAAQ,CAAC,MAAM;iBAC5B,CAAC,CAAC,CAAC;gBACJ,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,GAAG,EAAE,sBAAsB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBAC5E,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;gBAEtC,IAAI,CAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;oBAAC,EAAE,CAAC,KAAK,EAAE,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAA,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBAEhH,IAAI,CAAC;oBACH,OAAO,IAAI,EAAE,CAAC;wBACZ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACrB,MAAM,KAAK,CAAC,KAAK,EAAG,CAAC;4BACrB,SAAS;wBACX,CAAC;wBACD,IAAI,MAAM;4BAAE,MAAM;wBAClB,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAA6B,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;wBAC/F,IAAI,IAAI,CAAC,IAAI;4BAAE,MAAM;wBACrB,MAAM,IAAI,CAAC,KAAK,CAAC;oBACnB,CAAC;oBACD,IAAI,QAAQ;wBAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;gBAC/D,CAAC;wBAAS,CAAC;oBACT,QAAQ,CAAC,IAAI,GAAG;wBACd,UAAU,EAAE,SAAS;wBACrB,UAAU,EAAE,UAAU;wBACtB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;qBACnC,CAAC;gBACJ,CAAC;YACH,CAAC;SACF,CAAC;IACJ,CAAC;CACF;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,OAAe,EAAE,KAAa,EAAE,MAAwB,EAAE,UAAkB;IAC1G,MAAM,YAAY,GAAG,kBAAkB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAC5D,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;QACjC,QAAQ,EAAE,KAAK;QACf,aAAa,EAAE,YAAY;KAC5B,CAAC,CAAC;IACH,OAAO,GAAG,IAAI,sBAAsB,kBAAkB,CAAC,OAAO,CAAC,iBAAiB,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;AACtG,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAwB,EAAE,UAAkB;IACtE,IAAI,MAAM,KAAK,WAAW;QAAE,OAAO,WAAW,CAAC;IAC/C,IAAI,MAAM,KAAK,WAAW;QAAE,OAAO,WAAW,CAAC,CAAC,8BAA8B;IAC9E,uEAAuE;IACvE,MAAM,OAAO,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;IAC/D,OAAO,OAAO,IAAI,EAAE,CAAC;AACvB,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAwB;IACjD,IAAI,MAAM,KAAK,WAAW,IAAI,MAAM,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IAClE,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Text-to-speech provider surface. fabric-harness ships ElevenLabs and
|
|
3
|
+
* Cartesia implementations; bring-your-own for on-prem TTS or unsupported
|
|
4
|
+
* vendors. Used by `PipelineVoiceProvider` to drive the audio-out side of
|
|
5
|
+
* STT→LLM→TTS voice agents.
|
|
6
|
+
*
|
|
7
|
+
* For monolithic realtime models that own audio in *and* out (OpenAI
|
|
8
|
+
* Realtime), use `OpenAIRealtimeVoiceProvider` directly — TTS is internal.
|
|
9
|
+
*/
|
|
10
|
+
import type { VoiceAudioFormat } from './voice.js';
|
|
11
|
+
export interface TtsSynthesisOptions {
|
|
12
|
+
/** Voice timbre / cloned-voice id. Provider-specific. */
|
|
13
|
+
voice?: string;
|
|
14
|
+
/** Audio format on the wire. Default `pcm16`. */
|
|
15
|
+
audioFormat?: VoiceAudioFormat;
|
|
16
|
+
/** Sample rate in Hz. Default 24000 for `pcm16`, 8000 for telephony codecs. */
|
|
17
|
+
sampleRate?: number;
|
|
18
|
+
/** Optional speaking-rate / speed multiplier (1.0 = default). */
|
|
19
|
+
speed?: number;
|
|
20
|
+
/** Abort signal — terminates an in-flight synthesis cleanly. */
|
|
21
|
+
signal?: AbortSignal;
|
|
22
|
+
}
|
|
23
|
+
export interface TtsSynthesisUsage {
|
|
24
|
+
/** Characters of input text submitted (used for billing rollup). */
|
|
25
|
+
characters: number;
|
|
26
|
+
/** Total bytes of audio emitted. */
|
|
27
|
+
audioBytes: number;
|
|
28
|
+
/** Wall-clock duration of the synthesis call in milliseconds. */
|
|
29
|
+
durationMs: number;
|
|
30
|
+
}
|
|
31
|
+
export interface TtsProvider {
|
|
32
|
+
/** Stable provider id used in cost telemetry, e.g. `elevenlabs`, `cartesia`. */
|
|
33
|
+
readonly name: string;
|
|
34
|
+
/**
|
|
35
|
+
* Synthesize a complete utterance. Yields raw audio frames in
|
|
36
|
+
* `opts.audioFormat` (default `pcm16`). The final iteration value is
|
|
37
|
+
* the usage rollup — read it from `provider.lastUsage()` when the
|
|
38
|
+
* iterator completes.
|
|
39
|
+
*/
|
|
40
|
+
synthesize(text: string, opts?: TtsSynthesisOptions): AsyncIterable<Uint8Array>;
|
|
41
|
+
/**
|
|
42
|
+
* Optional: streaming text input. The provider should chunk audio out
|
|
43
|
+
* incrementally as text arrives — useful for low-latency LLM-token-to-
|
|
44
|
+
* audio paths. Falls back to buffering + `synthesize()` if absent.
|
|
45
|
+
*/
|
|
46
|
+
synthesizeStream?(text: AsyncIterable<string>, opts?: TtsSynthesisOptions): AsyncIterable<Uint8Array>;
|
|
47
|
+
/** Last completed synthesis usage. Reset at the start of each call. */
|
|
48
|
+
lastUsage(): TtsSynthesisUsage | undefined;
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=voice-tts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voice-tts.d.ts","sourceRoot":"","sources":["../src/voice-tts.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAEnD,MAAM,WAAW,mBAAmB;IAClC,yDAAyD;IACzD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iDAAiD;IACjD,WAAW,CAAC,EAAE,gBAAgB,CAAC;IAC/B,+EAA+E;IAC/E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gEAAgE;IAChE,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,iBAAiB;IAChC,oEAAoE;IACpE,UAAU,EAAE,MAAM,CAAC;IACnB,oCAAoC;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,WAAW;IAC1B,gFAAgF;IAChF,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;;;;OAKG;IACH,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,mBAAmB,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;IAChF;;;;OAIG;IACH,gBAAgB,CAAC,CAAC,IAAI,EAAE,aAAa,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,EAAE,mBAAmB,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;IACtG,uEAAuE;IACvE,SAAS,IAAI,iBAAiB,GAAG,SAAS,CAAC;CAC5C"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Text-to-speech provider surface. fabric-harness ships ElevenLabs and
|
|
3
|
+
* Cartesia implementations; bring-your-own for on-prem TTS or unsupported
|
|
4
|
+
* vendors. Used by `PipelineVoiceProvider` to drive the audio-out side of
|
|
5
|
+
* STT→LLM→TTS voice agents.
|
|
6
|
+
*
|
|
7
|
+
* For monolithic realtime models that own audio in *and* out (OpenAI
|
|
8
|
+
* Realtime), use `OpenAIRealtimeVoiceProvider` directly — TTS is internal.
|
|
9
|
+
*/
|
|
10
|
+
export {};
|
|
11
|
+
//# sourceMappingURL=voice-tts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voice-tts.js","sourceRoot":"","sources":["../src/voice-tts.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG"}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import type { ModelUsage } from './model.js';
|
|
2
|
+
import type { VoiceAudioFormat } from './voice.js';
|
|
3
|
+
/**
|
|
4
|
+
* Lightweight WebSocket client for the `fh server` `WS /sessions/:id/voice`
|
|
5
|
+
* endpoint. Server-side bridge owns the provider connection and API keys;
|
|
6
|
+
* this client just streams audio + control messages over WS.
|
|
7
|
+
*
|
|
8
|
+
* Works in browsers and on Node 22+ (uses the global `WebSocket`).
|
|
9
|
+
*/
|
|
10
|
+
export interface VoiceWsClientOptions {
|
|
11
|
+
/** WS URL — e.g. `ws://localhost:9111/sessions/abc/voice`. */
|
|
12
|
+
url: string;
|
|
13
|
+
/** Bearer token, sent as `?token=` query param. */
|
|
14
|
+
authToken?: string;
|
|
15
|
+
/** Tenant id, sent as `?tenant=` query param. */
|
|
16
|
+
tenantId?: string;
|
|
17
|
+
/** Realtime voice timbre id (alloy, nova, verse, ...). */
|
|
18
|
+
voice?: string;
|
|
19
|
+
/** Initial instructions / persona. */
|
|
20
|
+
instructions?: string;
|
|
21
|
+
/** Audio format hint. Default `pcm16` (server-side default). */
|
|
22
|
+
audioFormat?: VoiceAudioFormat;
|
|
23
|
+
/** Realtime model id — overrides server default. */
|
|
24
|
+
model?: string;
|
|
25
|
+
/** Called for every event from the server. */
|
|
26
|
+
onEvent?: (event: VoiceWsClientEvent) => void;
|
|
27
|
+
/** Called once when the bridge confirms the session. */
|
|
28
|
+
onSessionOpen?: (sessionId: string) => void;
|
|
29
|
+
/** Called on errors (transport or server-side). */
|
|
30
|
+
onError?: (error: Error) => void;
|
|
31
|
+
/** Called when the socket closes. */
|
|
32
|
+
onClose?: () => void;
|
|
33
|
+
}
|
|
34
|
+
export type VoiceWsClientEvent = {
|
|
35
|
+
type: 'session_open';
|
|
36
|
+
sessionId: string;
|
|
37
|
+
} | {
|
|
38
|
+
type: 'audio';
|
|
39
|
+
audio: Uint8Array;
|
|
40
|
+
} | {
|
|
41
|
+
type: 'text_delta';
|
|
42
|
+
delta: string;
|
|
43
|
+
role: 'assistant' | 'user';
|
|
44
|
+
} | {
|
|
45
|
+
type: 'transcript';
|
|
46
|
+
text: string;
|
|
47
|
+
role: 'assistant' | 'user';
|
|
48
|
+
} | {
|
|
49
|
+
type: 'tool_call';
|
|
50
|
+
id: string;
|
|
51
|
+
name: string;
|
|
52
|
+
input: unknown;
|
|
53
|
+
} | {
|
|
54
|
+
type: 'response_done';
|
|
55
|
+
usage?: ModelUsage;
|
|
56
|
+
} | {
|
|
57
|
+
type: 'cost_limit';
|
|
58
|
+
scope: string;
|
|
59
|
+
observedUsd: number;
|
|
60
|
+
limitUsd: number;
|
|
61
|
+
scopeKey?: string;
|
|
62
|
+
} | {
|
|
63
|
+
type: 'error';
|
|
64
|
+
message: string;
|
|
65
|
+
};
|
|
66
|
+
export interface VoiceWsClientHandle {
|
|
67
|
+
sendAudio(frame: Uint8Array): void;
|
|
68
|
+
sendText(text: string): void;
|
|
69
|
+
submitToolResult(input: {
|
|
70
|
+
id: string;
|
|
71
|
+
output: unknown;
|
|
72
|
+
}): void;
|
|
73
|
+
cancel(): void;
|
|
74
|
+
close(): void;
|
|
75
|
+
}
|
|
76
|
+
export declare function connectFabricVoice(options: VoiceWsClientOptions): VoiceWsClientHandle;
|
|
77
|
+
//# sourceMappingURL=voice-ws-client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voice-ws-client.d.ts","sourceRoot":"","sources":["../src/voice-ws-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAEnD;;;;;;GAMG;AAEH,MAAM,WAAW,oBAAoB;IACnC,8DAA8D;IAC9D,GAAG,EAAE,MAAM,CAAC;IACZ,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0DAA0D;IAC1D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gEAAgE;IAChE,WAAW,CAAC,EAAE,gBAAgB,CAAC;IAC/B,oDAAoD;IACpD,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,8CAA8C;IAC9C,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,kBAAkB,KAAK,IAAI,CAAC;IAC9C,wDAAwD;IACxD,aAAa,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5C,mDAAmD;IACnD,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IACjC,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;CACtB;AAED,MAAM,MAAM,kBAAkB,GAC1B;IAAE,IAAI,EAAE,cAAc,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC3C;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,UAAU,CAAA;CAAE,GACpC;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,WAAW,GAAG,MAAM,CAAA;CAAE,GACjE;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,WAAW,GAAG,MAAM,CAAA;CAAE,GAChE;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,GAC/D;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,KAAK,CAAC,EAAE,UAAU,CAAA;CAAE,GAC7C;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/F;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvC,MAAM,WAAW,mBAAmB;IAClC,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,gBAAgB,CAAC,KAAK,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;IAC/D,MAAM,IAAI,IAAI,CAAC;IACf,KAAK,IAAI,IAAI,CAAC;CACf;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,oBAAoB,GAAG,mBAAmB,CAkDrF"}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
export function connectFabricVoice(options) {
|
|
2
|
+
const url = appendQueryParams(options.url, {
|
|
3
|
+
...(options.authToken ? { token: options.authToken } : {}),
|
|
4
|
+
...(options.tenantId ? { tenant: options.tenantId } : {}),
|
|
5
|
+
...(options.voice ? { voice: options.voice } : {}),
|
|
6
|
+
...(options.instructions ? { instructions: options.instructions } : {}),
|
|
7
|
+
...(options.audioFormat ? { audioFormat: options.audioFormat } : {}),
|
|
8
|
+
...(options.model ? { model: options.model } : {}),
|
|
9
|
+
});
|
|
10
|
+
const WebSocketImpl = typeof globalThis.WebSocket === 'function'
|
|
11
|
+
? globalThis.WebSocket
|
|
12
|
+
: undefined;
|
|
13
|
+
if (!WebSocketImpl) {
|
|
14
|
+
throw new Error('connectFabricVoice: global WebSocket not available. On Node use Node 22+ or polyfill.');
|
|
15
|
+
}
|
|
16
|
+
const ws = new WebSocketImpl(url);
|
|
17
|
+
ws.addEventListener('message', (ev) => {
|
|
18
|
+
const data = typeof ev.data === 'string' ? ev.data : '';
|
|
19
|
+
if (!data)
|
|
20
|
+
return;
|
|
21
|
+
let parsed;
|
|
22
|
+
try {
|
|
23
|
+
parsed = JSON.parse(data);
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
options.onError?.(new Error(`Invalid JSON from voice bridge: ${data.slice(0, 120)}`));
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
const event = decodeServerMessage(parsed);
|
|
30
|
+
if (!event)
|
|
31
|
+
return;
|
|
32
|
+
options.onEvent?.(event);
|
|
33
|
+
if (event.type === 'session_open')
|
|
34
|
+
options.onSessionOpen?.(event.sessionId);
|
|
35
|
+
if (event.type === 'error')
|
|
36
|
+
options.onError?.(new Error(event.message));
|
|
37
|
+
});
|
|
38
|
+
ws.addEventListener('error', () => options.onError?.(new Error('WebSocket error')));
|
|
39
|
+
ws.addEventListener('close', () => options.onClose?.());
|
|
40
|
+
const send = (payload) => {
|
|
41
|
+
try {
|
|
42
|
+
if (ws.readyState === ws.OPEN)
|
|
43
|
+
ws.send(JSON.stringify(payload));
|
|
44
|
+
}
|
|
45
|
+
catch (error) {
|
|
46
|
+
options.onError?.(error instanceof Error ? error : new Error(String(error)));
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
return {
|
|
50
|
+
sendAudio(frame) { send({ type: 'audio', audio: bytesToBase64(frame) }); },
|
|
51
|
+
sendText(text) { send({ type: 'text', text }); },
|
|
52
|
+
submitToolResult(input) { send({ type: 'tool_result', id: input.id, output: input.output }); },
|
|
53
|
+
cancel() { send({ type: 'cancel' }); },
|
|
54
|
+
close() { try {
|
|
55
|
+
ws.close();
|
|
56
|
+
}
|
|
57
|
+
catch { } },
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
function decodeServerMessage(parsed) {
|
|
61
|
+
const type = String(parsed.type ?? '');
|
|
62
|
+
switch (type) {
|
|
63
|
+
case 'session_open':
|
|
64
|
+
return { type: 'session_open', sessionId: String(parsed.sessionId ?? '') };
|
|
65
|
+
case 'audio':
|
|
66
|
+
return typeof parsed.audio === 'string' ? { type: 'audio', audio: base64ToBytes(parsed.audio) } : undefined;
|
|
67
|
+
case 'text_delta':
|
|
68
|
+
return { type: 'text_delta', delta: String(parsed.delta ?? ''), role: (parsed.role === 'user' ? 'user' : 'assistant') };
|
|
69
|
+
case 'transcript':
|
|
70
|
+
return { type: 'transcript', text: String(parsed.text ?? ''), role: (parsed.role === 'user' ? 'user' : 'assistant') };
|
|
71
|
+
case 'tool_call':
|
|
72
|
+
return { type: 'tool_call', id: String(parsed.id ?? ''), name: String(parsed.name ?? ''), input: parsed.input };
|
|
73
|
+
case 'response_done':
|
|
74
|
+
return { type: 'response_done', ...(parsed.usage ? { usage: parsed.usage } : {}) };
|
|
75
|
+
case 'cost_limit':
|
|
76
|
+
return {
|
|
77
|
+
type: 'cost_limit',
|
|
78
|
+
scope: String(parsed.scope ?? ''),
|
|
79
|
+
observedUsd: Number(parsed.observedUsd ?? 0),
|
|
80
|
+
limitUsd: Number(parsed.limitUsd ?? 0),
|
|
81
|
+
...(typeof parsed.scopeKey === 'string' ? { scopeKey: parsed.scopeKey } : {}),
|
|
82
|
+
};
|
|
83
|
+
case 'error':
|
|
84
|
+
return { type: 'error', message: String(parsed.message ?? 'Voice bridge error') };
|
|
85
|
+
case 'pong':
|
|
86
|
+
case 'ack':
|
|
87
|
+
return undefined;
|
|
88
|
+
default:
|
|
89
|
+
return undefined;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
function appendQueryParams(url, params) {
|
|
93
|
+
const qs = new URLSearchParams(params).toString();
|
|
94
|
+
if (!qs)
|
|
95
|
+
return url;
|
|
96
|
+
return url.includes('?') ? `${url}&${qs}` : `${url}?${qs}`;
|
|
97
|
+
}
|
|
98
|
+
function bytesToBase64(bytes) {
|
|
99
|
+
if (typeof Buffer !== 'undefined')
|
|
100
|
+
return Buffer.from(bytes).toString('base64');
|
|
101
|
+
let binary = '';
|
|
102
|
+
for (let i = 0; i < bytes.byteLength; i += 1)
|
|
103
|
+
binary += String.fromCharCode(bytes[i]);
|
|
104
|
+
return btoa(binary);
|
|
105
|
+
}
|
|
106
|
+
function base64ToBytes(value) {
|
|
107
|
+
if (typeof Buffer !== 'undefined')
|
|
108
|
+
return new Uint8Array(Buffer.from(value, 'base64'));
|
|
109
|
+
const bin = atob(value);
|
|
110
|
+
const out = new Uint8Array(bin.length);
|
|
111
|
+
for (let i = 0; i < bin.length; i += 1)
|
|
112
|
+
out[i] = bin.charCodeAt(i);
|
|
113
|
+
return out;
|
|
114
|
+
}
|
|
115
|
+
//# sourceMappingURL=voice-ws-client.js.map
|