@livekit/agents-plugin-cartesia 1.6.2 → 1.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1 -1
- package/dist/index.js +1 -1
- package/dist/stt.cjs +51 -9
- package/dist/stt.cjs.map +1 -1
- package/dist/stt.d.ts.map +1 -1
- package/dist/stt.js +54 -9
- package/dist/stt.js.map +1 -1
- package/dist/stt.test.cjs +35 -0
- package/dist/stt.test.cjs.map +1 -1
- package/dist/stt.test.js +38 -3
- package/dist/stt.test.js.map +1 -1
- package/dist/tts.cjs +37 -4
- package/dist/tts.cjs.map +1 -1
- package/dist/tts.js +37 -4
- package/dist/tts.js.map +1 -1
- package/dist/tts.test.cjs +49 -0
- package/dist/tts.test.cjs.map +1 -1
- package/dist/tts.test.js +55 -1
- package/dist/tts.test.js.map +1 -1
- package/package.json +6 -6
- package/src/stt.test.ts +42 -3
- package/src/stt.ts +56 -9
- package/src/tts.test.ts +62 -1
- package/src/tts.ts +42 -4
package/src/stt.test.ts
CHANGED
|
@@ -1,13 +1,22 @@
|
|
|
1
1
|
// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
|
|
2
2
|
//
|
|
3
3
|
// SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
import { APIConnectionError, APIError, DEFAULT_API_CONNECT_OPTIONS, stt } from '@livekit/agents';
|
|
4
5
|
import { VAD } from '@livekit/agents-plugin-silero';
|
|
5
|
-
import { stt } from '@livekit/agents-plugins-test';
|
|
6
|
-
import {
|
|
6
|
+
import { stt as testStt } from '@livekit/agents-plugins-test';
|
|
7
|
+
import { once } from 'node:events';
|
|
8
|
+
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
|
7
9
|
import { STT } from './stt.js';
|
|
8
10
|
|
|
9
11
|
const hasCartesiaApiKey = Boolean(process.env.CARTESIA_API_KEY);
|
|
10
12
|
|
|
13
|
+
const swallowExpectedRejection = (reason: unknown) => {
|
|
14
|
+
if (reason instanceof APIError) return;
|
|
15
|
+
throw reason;
|
|
16
|
+
};
|
|
17
|
+
beforeAll(() => process.on('unhandledRejection', swallowExpectedRejection));
|
|
18
|
+
afterAll(() => void process.off('unhandledRejection', swallowExpectedRejection));
|
|
19
|
+
|
|
11
20
|
describe('Cartesia STT capabilities', () => {
|
|
12
21
|
it('reports no aligned transcript for Ink-2', () => {
|
|
13
22
|
const instance = new STT({ apiKey: 'test-key', model: 'ink-2' });
|
|
@@ -16,9 +25,39 @@ describe('Cartesia STT capabilities', () => {
|
|
|
16
25
|
});
|
|
17
26
|
});
|
|
18
27
|
|
|
28
|
+
describe('Cartesia STT connection errors', () => {
|
|
29
|
+
it('does not retain synchronous WebSocket connection errors', async () => {
|
|
30
|
+
const secret = 'cartesia-secret-api-key-do-not-log';
|
|
31
|
+
const cartesia = new STT({ apiKey: 'test-key', baseUrl: `http://[${secret}` });
|
|
32
|
+
const errorEvent = once(cartesia, 'error') as Promise<Parameters<stt.STTCallbacks['error']>>;
|
|
33
|
+
const stream = cartesia.stream({
|
|
34
|
+
connOptions: { ...DEFAULT_API_CONNECT_OPTIONS, maxRetry: 0 },
|
|
35
|
+
});
|
|
36
|
+
const drain = (async () => {
|
|
37
|
+
for await (const _ of stream) {
|
|
38
|
+
// discard events
|
|
39
|
+
}
|
|
40
|
+
})();
|
|
41
|
+
|
|
42
|
+
try {
|
|
43
|
+
const [{ error }] = await errorEvent;
|
|
44
|
+
expect(error).toBeInstanceOf(APIConnectionError);
|
|
45
|
+
expect(error.message).toBe('SyntaxError');
|
|
46
|
+
expect(error.message).not.toContain(secret);
|
|
47
|
+
expect(error.toString()).not.toContain(secret);
|
|
48
|
+
expect((error as Error & { cause?: unknown }).cause).toBeUndefined();
|
|
49
|
+
} finally {
|
|
50
|
+
stream.close();
|
|
51
|
+
await drain.catch(() => {});
|
|
52
|
+
// Let SpeechStream.mainTask's expected rejection reach the file-level handler.
|
|
53
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
54
|
+
}
|
|
55
|
+
});
|
|
56
|
+
});
|
|
57
|
+
|
|
19
58
|
if (hasCartesiaApiKey) {
|
|
20
59
|
describe('Cartesia STT', async () => {
|
|
21
|
-
await
|
|
60
|
+
await testStt(new STT(), await VAD.load(), { nonStreaming: false });
|
|
22
61
|
});
|
|
23
62
|
} else {
|
|
24
63
|
describe('Cartesia STT', () => {
|
package/src/stt.ts
CHANGED
|
@@ -4,7 +4,10 @@
|
|
|
4
4
|
import {
|
|
5
5
|
type APIConnectOptions,
|
|
6
6
|
APIConnectionError,
|
|
7
|
+
APIStatusError,
|
|
7
8
|
AudioByteStream,
|
|
9
|
+
DEFAULT_API_CONNECT_OPTIONS,
|
|
10
|
+
asError,
|
|
8
11
|
asLanguageCode,
|
|
9
12
|
calculateAudioDurationSeconds,
|
|
10
13
|
getBaseLanguage,
|
|
@@ -25,6 +28,14 @@ const API_VERSION = '2026-03-01';
|
|
|
25
28
|
const DRAIN_TIMEOUT_MS = 5000;
|
|
26
29
|
const KEEPALIVE_INTERVAL_MS = 30000;
|
|
27
30
|
|
|
31
|
+
const sanitizedErrorName = (error: Error): string => {
|
|
32
|
+
if (error instanceof SyntaxError) return 'SyntaxError';
|
|
33
|
+
if (error instanceof TypeError) return 'TypeError';
|
|
34
|
+
if (error instanceof RangeError) return 'RangeError';
|
|
35
|
+
if (error instanceof AggregateError) return 'AggregateError';
|
|
36
|
+
return 'Error';
|
|
37
|
+
};
|
|
38
|
+
|
|
28
39
|
/**
|
|
29
40
|
* Fires once when the WebSocket connection is established.
|
|
30
41
|
*
|
|
@@ -267,10 +278,12 @@ export class SpeechStream extends stt.SpeechStream {
|
|
|
267
278
|
#currentTranscript = '';
|
|
268
279
|
#speechDuration = 0;
|
|
269
280
|
#closingWs = false;
|
|
281
|
+
#connectTimeout: number;
|
|
270
282
|
|
|
271
283
|
constructor(sttInstance: STT, opts: STTOptions, connOptions?: APIConnectOptions) {
|
|
272
284
|
super(sttInstance, opts.sampleRate, connOptions);
|
|
273
285
|
this.#opts = { ...opts };
|
|
286
|
+
this.#connectTimeout = connOptions?.timeoutMs ?? DEFAULT_API_CONNECT_OPTIONS.timeoutMs;
|
|
274
287
|
}
|
|
275
288
|
|
|
276
289
|
override get label(): string {
|
|
@@ -295,12 +308,21 @@ export class SpeechStream extends stt.SpeechStream {
|
|
|
295
308
|
const url = this.#getCartesiaUrl();
|
|
296
309
|
this.#logger.debug(`Connecting to Cartesia STT: ${url}`);
|
|
297
310
|
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
311
|
+
let ws: WebSocket;
|
|
312
|
+
try {
|
|
313
|
+
ws = new WebSocket(url, {
|
|
314
|
+
handshakeTimeout: this.#connectTimeout,
|
|
315
|
+
headers: {
|
|
316
|
+
[AUTHORIZATION_HEADER]: this.#opts.apiKey,
|
|
317
|
+
[VERSION_HEADER]: API_VERSION,
|
|
318
|
+
},
|
|
319
|
+
});
|
|
320
|
+
} catch (error) {
|
|
321
|
+
throw new APIConnectionError({
|
|
322
|
+
message: sanitizedErrorName(asError(error)),
|
|
323
|
+
options: { retryable: true },
|
|
324
|
+
});
|
|
325
|
+
}
|
|
304
326
|
this.#ws = ws;
|
|
305
327
|
|
|
306
328
|
// Cartesia returns the request id on the WS upgrade response, before any
|
|
@@ -316,16 +338,41 @@ export class SpeechStream extends stt.SpeechStream {
|
|
|
316
338
|
});
|
|
317
339
|
|
|
318
340
|
await new Promise<void>((resolve, reject) => {
|
|
319
|
-
const
|
|
341
|
+
const cleanup = () => {
|
|
342
|
+
ws.off('open', onOpen);
|
|
343
|
+
ws.off('unexpected-response', onUnexpectedResponse);
|
|
320
344
|
ws.off('error', onError);
|
|
345
|
+
};
|
|
346
|
+
const onOpen = () => {
|
|
347
|
+
cleanup();
|
|
321
348
|
resolve();
|
|
322
349
|
};
|
|
350
|
+
const onUnexpectedResponse = (_request: unknown, response: { statusCode?: number }) => {
|
|
351
|
+
cleanup();
|
|
352
|
+
ws.on('error', () => {});
|
|
353
|
+
ws.close();
|
|
354
|
+
// Authentication headers can appear in WebSocket handshake errors.
|
|
355
|
+
const statusCode = response.statusCode ?? -1;
|
|
356
|
+
reject(
|
|
357
|
+
new APIStatusError({
|
|
358
|
+
message: `Cartesia WebSocket connection rejected with status ${statusCode}`,
|
|
359
|
+
options: { statusCode },
|
|
360
|
+
}),
|
|
361
|
+
);
|
|
362
|
+
};
|
|
323
363
|
const onError = (err: Error) => {
|
|
324
|
-
|
|
325
|
-
|
|
364
|
+
cleanup();
|
|
365
|
+
// Transport errors can contain credentials in URLs.
|
|
366
|
+
reject(
|
|
367
|
+
new APIConnectionError({
|
|
368
|
+
message: sanitizedErrorName(err),
|
|
369
|
+
options: { retryable: true },
|
|
370
|
+
}),
|
|
371
|
+
);
|
|
326
372
|
};
|
|
327
373
|
|
|
328
374
|
ws.once('open', onOpen);
|
|
375
|
+
ws.once('unexpected-response', onUnexpectedResponse);
|
|
329
376
|
ws.once('error', onError);
|
|
330
377
|
});
|
|
331
378
|
|
package/src/tts.test.ts
CHANGED
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
|
|
2
2
|
//
|
|
3
3
|
// SPDX-License-Identifier: Apache-2.0
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
type APIConnectOptions,
|
|
6
|
+
APIConnectionError,
|
|
7
|
+
APIStatusError,
|
|
8
|
+
DEFAULT_API_CONNECT_OPTIONS,
|
|
9
|
+
tts,
|
|
10
|
+
} from '@livekit/agents';
|
|
5
11
|
import { STT } from '@livekit/agents-plugin-openai';
|
|
6
12
|
import { tts as testTts } from '@livekit/agents-plugins-test';
|
|
7
13
|
import { once } from 'node:events';
|
|
@@ -110,6 +116,61 @@ async function synthesizeTurn(
|
|
|
110
116
|
}
|
|
111
117
|
|
|
112
118
|
describe('Cartesia streaming pool', () => {
|
|
119
|
+
it('redacts API keys from WebSocket handshake errors', async () => {
|
|
120
|
+
const secret = 'cartesia-secret-api-key-do-not-log';
|
|
121
|
+
const wss = new WebSocketServer({
|
|
122
|
+
host: '127.0.0.1',
|
|
123
|
+
port: 0,
|
|
124
|
+
verifyClient: (_info, done) => done(false, 401, 'Unauthorized'),
|
|
125
|
+
});
|
|
126
|
+
await once(wss, 'listening');
|
|
127
|
+
const address = wss.address() as AddressInfo;
|
|
128
|
+
const cartesia = new TTS({ apiKey: secret, baseUrl: `http://127.0.0.1:${address.port}` });
|
|
129
|
+
const errorEvent = once(cartesia, 'error') as Promise<Parameters<tts.TTSCallbacks['error']>>;
|
|
130
|
+
|
|
131
|
+
try {
|
|
132
|
+
const stream = cartesia.stream({
|
|
133
|
+
connOptions: { ...DEFAULT_API_CONNECT_OPTIONS, maxRetry: 0 },
|
|
134
|
+
});
|
|
135
|
+
stream.pushText('test');
|
|
136
|
+
stream.endInput();
|
|
137
|
+
|
|
138
|
+
const [{ error }] = await errorEvent;
|
|
139
|
+
expect(error).toBeInstanceOf(APIStatusError);
|
|
140
|
+
expect((error as APIStatusError).statusCode).toBe(401);
|
|
141
|
+
expect(error.message).not.toContain(secret);
|
|
142
|
+
expect(error.toString()).not.toContain(secret);
|
|
143
|
+
stream.close();
|
|
144
|
+
} finally {
|
|
145
|
+
await cartesia.close();
|
|
146
|
+
await closeWebSocketServer(wss);
|
|
147
|
+
}
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
it('does not retain generic WebSocket connection errors', async () => {
|
|
151
|
+
const secret = 'cartesia-secret-api-key-do-not-log';
|
|
152
|
+
const cartesia = new TTS({ apiKey: secret, baseUrl: `http://[${secret}` });
|
|
153
|
+
const errorEvent = once(cartesia, 'error') as Promise<Parameters<tts.TTSCallbacks['error']>>;
|
|
154
|
+
|
|
155
|
+
try {
|
|
156
|
+
const stream = cartesia.stream({
|
|
157
|
+
connOptions: { ...DEFAULT_API_CONNECT_OPTIONS, maxRetry: 0 },
|
|
158
|
+
});
|
|
159
|
+
stream.pushText('test');
|
|
160
|
+
stream.endInput();
|
|
161
|
+
|
|
162
|
+
const [{ error }] = await errorEvent;
|
|
163
|
+
expect(error).toBeInstanceOf(APIConnectionError);
|
|
164
|
+
expect(error.message).toBe('SyntaxError');
|
|
165
|
+
expect(error.message).not.toContain(secret);
|
|
166
|
+
expect(error.toString()).not.toContain(secret);
|
|
167
|
+
expect((error as Error & { cause?: unknown }).cause).toBeUndefined();
|
|
168
|
+
stream.close();
|
|
169
|
+
} finally {
|
|
170
|
+
await cartesia.close();
|
|
171
|
+
}
|
|
172
|
+
});
|
|
173
|
+
|
|
113
174
|
it('reuses one websocket across sequential turns', async () => {
|
|
114
175
|
const { wss, baseURL } = await startWebSocketServer();
|
|
115
176
|
const server = serveCartesia(wss);
|
package/src/tts.ts
CHANGED
|
@@ -710,6 +710,14 @@ const isRecord = (v: unknown): v is Record<string, unknown> => {
|
|
|
710
710
|
return v !== null && typeof v === 'object';
|
|
711
711
|
};
|
|
712
712
|
|
|
713
|
+
const sanitizedErrorName = (error: Error): string => {
|
|
714
|
+
if (error instanceof SyntaxError) return 'SyntaxError';
|
|
715
|
+
if (error instanceof TypeError) return 'TypeError';
|
|
716
|
+
if (error instanceof RangeError) return 'RangeError';
|
|
717
|
+
if (error instanceof AggregateError) return 'AggregateError';
|
|
718
|
+
return 'Error';
|
|
719
|
+
};
|
|
720
|
+
|
|
713
721
|
const isAggregateErrorLike = (e: unknown): e is { errors: unknown[]; name?: string } => {
|
|
714
722
|
if (!isRecord(e)) return false;
|
|
715
723
|
return e.name === 'AggregateError' && Array.isArray(e.errors);
|
|
@@ -763,6 +771,7 @@ const waitForWsOpen = async ({
|
|
|
763
771
|
const cleanup = () => {
|
|
764
772
|
if (timeout) clearTimeout(timeout);
|
|
765
773
|
ws.off('open', onOpen);
|
|
774
|
+
ws.off('unexpected-response', onUnexpectedResponse);
|
|
766
775
|
ws.off('error', onError);
|
|
767
776
|
ws.off('close', onClose);
|
|
768
777
|
abortSignal?.removeEventListener('abort', onAbort);
|
|
@@ -770,6 +779,16 @@ const waitForWsOpen = async ({
|
|
|
770
779
|
|
|
771
780
|
const onOpen = () => fut.resolve();
|
|
772
781
|
const onError = (err: Error) => fut.reject(asError(err));
|
|
782
|
+
const onUnexpectedResponse = (_request: unknown, response: { statusCode?: number }) => {
|
|
783
|
+
// Authentication headers can appear in WebSocket handshake errors.
|
|
784
|
+
const statusCode = response.statusCode ?? -1;
|
|
785
|
+
fut.reject(
|
|
786
|
+
new APIStatusError({
|
|
787
|
+
message: `Cartesia WebSocket connection rejected with status ${statusCode}`,
|
|
788
|
+
options: { statusCode },
|
|
789
|
+
}),
|
|
790
|
+
);
|
|
791
|
+
};
|
|
773
792
|
const onClose = (code: number, reason: Buffer) =>
|
|
774
793
|
fut.reject(
|
|
775
794
|
new Error(`WebSocket closed before open (code=${code}, reason=${reason.toString()})`),
|
|
@@ -777,12 +796,16 @@ const waitForWsOpen = async ({
|
|
|
777
796
|
const onAbort = () => fut.reject(new Error('aborted'));
|
|
778
797
|
|
|
779
798
|
ws.on('open', onOpen);
|
|
799
|
+
ws.on('unexpected-response', onUnexpectedResponse);
|
|
780
800
|
ws.on('error', onError);
|
|
781
801
|
ws.on('close', onClose);
|
|
782
802
|
abortSignal?.addEventListener('abort', onAbort, { once: true });
|
|
783
803
|
|
|
784
804
|
if (timeoutMs > 0) {
|
|
785
|
-
timeout = setTimeout(
|
|
805
|
+
timeout = setTimeout(
|
|
806
|
+
() => fut.reject(new APITimeoutError({ message: 'Cartesia WebSocket connection timed out' })),
|
|
807
|
+
timeoutMs,
|
|
808
|
+
);
|
|
786
809
|
}
|
|
787
810
|
|
|
788
811
|
try {
|
|
@@ -859,9 +882,11 @@ const connectCartesiaWebSocket = async ({
|
|
|
859
882
|
}
|
|
860
883
|
};
|
|
861
884
|
|
|
885
|
+
let connectError: unknown;
|
|
862
886
|
try {
|
|
863
887
|
return await connectOnce();
|
|
864
888
|
} catch (e) {
|
|
889
|
+
connectError = e;
|
|
865
890
|
// Mitigation for Node.js dual-stack (IPv6/IPv4) connect flakiness ("happy eyeballs"):
|
|
866
891
|
// some environments surface `AggregateError` with nested `ETIMEDOUT` during the initial
|
|
867
892
|
// WebSocket open. In that case we do a one-off retry forcing IPv4 (`family: 4`) before
|
|
@@ -871,11 +896,24 @@ const connectCartesiaWebSocket = async ({
|
|
|
871
896
|
// - Increase the session TTS connect timeout (`connOptions.ttsConnOptions.timeoutMs`)
|
|
872
897
|
// - Or adjust Node's family autoselection behavior via `NODE_OPTIONS`, e.g.
|
|
873
898
|
// `--network-family-autoselection-attempt-timeout=5000` (or disable it entirely).
|
|
874
|
-
if (hasAnyTransientCode(e) || isAggregateErrorLike(e)) {
|
|
875
|
-
|
|
899
|
+
if (!(e instanceof APIError) && (hasAnyTransientCode(e) || isAggregateErrorLike(e))) {
|
|
900
|
+
try {
|
|
901
|
+
return await connectOnce(4);
|
|
902
|
+
} catch (retryError) {
|
|
903
|
+
connectError = retryError;
|
|
904
|
+
}
|
|
876
905
|
}
|
|
877
|
-
throw e;
|
|
878
906
|
}
|
|
907
|
+
|
|
908
|
+
if (connectError instanceof APIError) throw connectError;
|
|
909
|
+
const error = asError(connectError);
|
|
910
|
+
const isTimeout =
|
|
911
|
+
hasErrorCode(connectError, 'ETIMEDOUT') || /timed?\s*out|timeout/i.test(error.message);
|
|
912
|
+
if (isTimeout) {
|
|
913
|
+
throw new APITimeoutError({ message: 'Cartesia WebSocket connection timed out' });
|
|
914
|
+
}
|
|
915
|
+
// Transport errors can contain credentials in URLs.
|
|
916
|
+
throw new APIConnectionError({ message: sanitizedErrorName(error) });
|
|
879
917
|
};
|
|
880
918
|
|
|
881
919
|
const toCartesiaOptions = (
|