@kuralle-syrinx/cartesia 2.1.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -3
- package/src/index.test.ts +0 -128
- package/src/index.ts +133 -304
package/package.json
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kuralle-syrinx/cartesia",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "3.0.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"main": "./src/index.ts",
|
|
8
8
|
"types": "./src/index.ts",
|
|
9
9
|
"dependencies": {
|
|
10
|
-
"@kuralle-syrinx/core": "
|
|
11
|
-
"@kuralle-syrinx/
|
|
10
|
+
"@kuralle-syrinx/core": "3.0.0",
|
|
11
|
+
"@kuralle-syrinx/tts-core": "3.0.0",
|
|
12
|
+
"@kuralle-syrinx/ws": "3.0.0"
|
|
12
13
|
},
|
|
13
14
|
"devDependencies": {
|
|
14
15
|
"@types/ws": "^8.5.0",
|
package/src/index.test.ts
CHANGED
|
@@ -695,67 +695,6 @@ describe("CartesiaTTSPlugin", () => {
|
|
|
695
695
|
await started;
|
|
696
696
|
});
|
|
697
697
|
|
|
698
|
-
it("does not keep a Cartesia context active when the initial provider send fails", async () => {
|
|
699
|
-
const endpointUrl = await createLocalServer(() => {});
|
|
700
|
-
const bus = new PipelineBusImpl();
|
|
701
|
-
const started = startBus(bus);
|
|
702
|
-
const plugin = new CartesiaTTSPlugin();
|
|
703
|
-
const errors: TtsErrorPacket[] = [];
|
|
704
|
-
const ends: TextToSpeechEndPacket[] = [];
|
|
705
|
-
bus.on("tts.error", (pkt) => {
|
|
706
|
-
errors.push(pkt as TtsErrorPacket);
|
|
707
|
-
});
|
|
708
|
-
bus.on("tts.end", (pkt) => {
|
|
709
|
-
ends.push(pkt as TextToSpeechEndPacket);
|
|
710
|
-
});
|
|
711
|
-
|
|
712
|
-
await plugin.initialize(bus, {
|
|
713
|
-
api_key: "test-cartesia-key",
|
|
714
|
-
endpoint_url: endpointUrl,
|
|
715
|
-
voice_id: "voice-test",
|
|
716
|
-
model_id: "sonic-test",
|
|
717
|
-
});
|
|
718
|
-
// Simulate a closed socket: the managed connection's send throws.
|
|
719
|
-
const send = vi.fn(() => {
|
|
720
|
-
throw new Error("WebSocket is not open");
|
|
721
|
-
});
|
|
722
|
-
Object.assign(plugin as unknown as { conn: { ensureReady: () => Promise<void>; send: typeof send; close: () => Promise<void> } }, {
|
|
723
|
-
conn: { ensureReady: async () => undefined, send, close: async () => undefined },
|
|
724
|
-
});
|
|
725
|
-
bus.push(Route.Main, {
|
|
726
|
-
kind: "tts.text",
|
|
727
|
-
contextId: "turn-unsent",
|
|
728
|
-
timestampMs: Date.now(),
|
|
729
|
-
text: "This request never reaches Cartesia.",
|
|
730
|
-
});
|
|
731
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
732
|
-
bus.push(Route.Main, {
|
|
733
|
-
kind: "tts.done",
|
|
734
|
-
contextId: "turn-unsent",
|
|
735
|
-
timestampMs: Date.now(),
|
|
736
|
-
});
|
|
737
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
738
|
-
|
|
739
|
-
// The send was attempted but failed; the context must not stay active —
|
|
740
|
-
// an error fires and the turn ends instead of hanging.
|
|
741
|
-
expect(send).toHaveBeenCalled();
|
|
742
|
-
expect(errors).toEqual([
|
|
743
|
-
expect.objectContaining({
|
|
744
|
-
kind: "tts.error",
|
|
745
|
-
contextId: "turn-unsent",
|
|
746
|
-
component: "tts",
|
|
747
|
-
cause: expect.objectContaining({
|
|
748
|
-
message: "WebSocket is not open",
|
|
749
|
-
}),
|
|
750
|
-
}),
|
|
751
|
-
]);
|
|
752
|
-
expect(ends).toEqual([expect.objectContaining({ contextId: "turn-unsent" })]);
|
|
753
|
-
|
|
754
|
-
await plugin.close();
|
|
755
|
-
bus.stop();
|
|
756
|
-
await started;
|
|
757
|
-
});
|
|
758
|
-
|
|
759
698
|
it("emits tts.word_timestamps from Cartesia's real top-level parallel-array timestamp frame", async () => {
|
|
760
699
|
const audioChunk = new Uint8Array(3200);
|
|
761
700
|
const endpointUrl = await createLocalServer((socket) => {
|
|
@@ -814,71 +753,4 @@ describe("CartesiaTTSPlugin", () => {
|
|
|
814
753
|
await started;
|
|
815
754
|
});
|
|
816
755
|
|
|
817
|
-
it("does not keep a Cartesia context active when the terminal provider send fails", async () => {
|
|
818
|
-
const endpointUrl = await createLocalServer(() => {});
|
|
819
|
-
const bus = new PipelineBusImpl();
|
|
820
|
-
const started = startBus(bus);
|
|
821
|
-
const plugin = new CartesiaTTSPlugin();
|
|
822
|
-
const errors: TtsErrorPacket[] = [];
|
|
823
|
-
bus.on("tts.error", (pkt) => {
|
|
824
|
-
errors.push(pkt as TtsErrorPacket);
|
|
825
|
-
});
|
|
826
|
-
|
|
827
|
-
await plugin.initialize(bus, {
|
|
828
|
-
api_key: "test-cartesia-key",
|
|
829
|
-
endpoint_url: endpointUrl,
|
|
830
|
-
voice_id: "voice-test",
|
|
831
|
-
model_id: "sonic-test",
|
|
832
|
-
});
|
|
833
|
-
// The first send (the transcript) succeeds; the terminal flush send fails.
|
|
834
|
-
let failNext = false;
|
|
835
|
-
const send = vi.fn(() => {
|
|
836
|
-
if (failNext) throw new Error("WebSocket is not open");
|
|
837
|
-
});
|
|
838
|
-
Object.assign(plugin as unknown as { conn: { ensureReady: () => Promise<void>; send: typeof send; close: () => Promise<void> } }, {
|
|
839
|
-
conn: { ensureReady: async () => undefined, send, close: async () => undefined },
|
|
840
|
-
});
|
|
841
|
-
bus.push(Route.Main, {
|
|
842
|
-
kind: "tts.text",
|
|
843
|
-
contextId: "turn-terminal-unsent",
|
|
844
|
-
timestampMs: Date.now(),
|
|
845
|
-
text: "This request reaches Cartesia.",
|
|
846
|
-
});
|
|
847
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
848
|
-
expect(send).toHaveBeenCalledTimes(1);
|
|
849
|
-
|
|
850
|
-
failNext = true;
|
|
851
|
-
bus.push(Route.Main, {
|
|
852
|
-
kind: "tts.done",
|
|
853
|
-
contextId: "turn-terminal-unsent",
|
|
854
|
-
timestampMs: Date.now(),
|
|
855
|
-
});
|
|
856
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
857
|
-
expect(errors).toEqual([
|
|
858
|
-
expect.objectContaining({
|
|
859
|
-
kind: "tts.error",
|
|
860
|
-
contextId: "turn-terminal-unsent",
|
|
861
|
-
component: "tts",
|
|
862
|
-
cause: expect.objectContaining({
|
|
863
|
-
message: "WebSocket is not open",
|
|
864
|
-
}),
|
|
865
|
-
}),
|
|
866
|
-
]);
|
|
867
|
-
expect(send).toHaveBeenCalledTimes(2);
|
|
868
|
-
|
|
869
|
-
failNext = false;
|
|
870
|
-
bus.push(Route.Critical, {
|
|
871
|
-
kind: "interrupt.tts",
|
|
872
|
-
contextId: "turn-terminal-unsent",
|
|
873
|
-
timestampMs: Date.now(),
|
|
874
|
-
});
|
|
875
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
876
|
-
// The context was already removed, so the interrupt sends no Cancel.
|
|
877
|
-
expect(send).toHaveBeenCalledTimes(2);
|
|
878
|
-
|
|
879
|
-
await plugin.close();
|
|
880
|
-
bus.stop();
|
|
881
|
-
await started;
|
|
882
|
-
});
|
|
883
|
-
|
|
884
756
|
});
|
package/src/index.ts
CHANGED
|
@@ -1,367 +1,178 @@
|
|
|
1
1
|
// SPDX-License-Identifier: MIT
|
|
2
2
|
//
|
|
3
3
|
// Syrinx Kernel v2 — Cartesia TTS Plugin
|
|
4
|
+
//
|
|
5
|
+
// The streaming lifecycle lives in @kuralle-syrinx/tts-core. This file is the Cartesia wire
|
|
6
|
+
// protocol: per-context attribution (the provider echoes `context_id`), the speak/flush/cancel
|
|
7
|
+
// JSON frames, and the inbound decode of base64 audio + word timestamps + the `done` flag.
|
|
8
|
+
// Cartesia ships whole PCM16 frames, so it validates alignment in `decode` (erroring on an
|
|
9
|
+
// odd-length payload) rather than relying on the engine's streaming carry.
|
|
4
10
|
|
|
5
|
-
import type { PipelineBus } from "@kuralle-syrinx/core";
|
|
6
11
|
import {
|
|
7
12
|
Route,
|
|
8
|
-
type AudioFormat,
|
|
9
|
-
type PluginConfig,
|
|
10
|
-
type RetryConfig,
|
|
11
|
-
type TextToSpeechAudioPacket,
|
|
12
|
-
type TextToSpeechEndPacket,
|
|
13
|
-
type TextToSpeechWordTimestampsPacket,
|
|
14
|
-
type TtsWordTimestamp,
|
|
15
|
-
type TtsErrorPacket,
|
|
16
|
-
type VoicePlugin,
|
|
17
13
|
assertAudioFormat,
|
|
18
14
|
assertAudioPayload,
|
|
19
|
-
categorizeTtsError,
|
|
20
|
-
isRecoverable,
|
|
21
15
|
optionalStringConfig,
|
|
22
16
|
readProviderRetryConfig,
|
|
23
17
|
requireStringConfig,
|
|
18
|
+
type AudioFormat,
|
|
19
|
+
type PipelineBus,
|
|
20
|
+
type PluginConfig,
|
|
21
|
+
type TextToSpeechWordTimestampsPacket,
|
|
22
|
+
type TtsWordTimestamp,
|
|
23
|
+
type VoicePlugin,
|
|
24
24
|
} from "@kuralle-syrinx/core";
|
|
25
|
-
import {
|
|
25
|
+
import {
|
|
26
|
+
attributionKey,
|
|
27
|
+
defaultNodeSocketFactory,
|
|
28
|
+
startStreamingTtsSession,
|
|
29
|
+
type AttributionKey,
|
|
30
|
+
type StreamingTtsSession,
|
|
31
|
+
type WireEvent,
|
|
32
|
+
type WireProtocol,
|
|
33
|
+
} from "@kuralle-syrinx/tts-core";
|
|
34
|
+
import type { SocketData, SocketFactory } from "@kuralle-syrinx/ws";
|
|
26
35
|
|
|
27
36
|
const KEEP_ALIVE_INTERVAL_MS = 10_000;
|
|
28
37
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
private apiKey = "";
|
|
37
|
-
private voiceId = "c2ac25f9-ecc4-4f56-9095-651354df60c0";
|
|
38
|
-
private modelId = "sonic-3";
|
|
39
|
-
private endpointUrl = "wss://api.cartesia.ai/tts/websocket";
|
|
40
|
-
private apiVersion = "2024-06-10";
|
|
41
|
-
private sampleRate = 16000;
|
|
42
|
-
private language = "en";
|
|
43
|
-
private retryConfig: RetryConfig = readProviderRetryConfig({});
|
|
44
|
-
private activeContexts = new Set<string>();
|
|
45
|
-
private cancelledContexts = new Set<string>();
|
|
46
|
-
private finishTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
47
|
-
private disposers: Array<() => void> = [];
|
|
48
|
-
private audioFormat: AudioFormat = { encoding: "pcm_s16le", sampleRateHz: 16000, channels: 1 };
|
|
49
|
-
|
|
50
|
-
async initialize(bus: PipelineBus, config: PluginConfig): Promise<void> {
|
|
51
|
-
this.bus = bus;
|
|
52
|
-
this.apiKey = requireStringConfig(config, "api_key");
|
|
53
|
-
this.voiceId = optionalStringConfig(config, "voice_id") ?? this.voiceId;
|
|
54
|
-
this.modelId = optionalStringConfig(config, "model_id") ?? this.modelId;
|
|
55
|
-
this.endpointUrl = optionalStringConfig(config, "endpoint_url") ?? this.endpointUrl;
|
|
56
|
-
this.apiVersion = optionalStringConfig(config, "cartesia_version") ?? this.apiVersion;
|
|
57
|
-
this.sampleRate = (config["sample_rate"] as number) ?? this.sampleRate;
|
|
58
|
-
this.language = optionalStringConfig(config, "language") ?? this.language;
|
|
59
|
-
this.retryConfig = readProviderRetryConfig(config);
|
|
60
|
-
const finishTimeoutMs = readNonNegativeInteger(config["finish_timeout_ms"], 2000);
|
|
61
|
-
this.audioFormat = { encoding: "pcm_s16le", sampleRateHz: this.sampleRate, channels: 1 };
|
|
62
|
-
assertAudioFormat(this.audioFormat);
|
|
38
|
+
interface CartesiaWireConfig {
|
|
39
|
+
readonly modelId: string;
|
|
40
|
+
readonly voiceId: string;
|
|
41
|
+
readonly sampleRate: number;
|
|
42
|
+
readonly language: string;
|
|
43
|
+
readonly audioFormat: AudioFormat;
|
|
44
|
+
}
|
|
63
45
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
const params = new URLSearchParams({ cartesia_version: this.apiVersion });
|
|
67
|
-
const separator = this.endpointUrl.includes("?") ? "&" : "?";
|
|
68
|
-
return `${this.endpointUrl}${separator}${params.toString()}`;
|
|
69
|
-
},
|
|
70
|
-
headers: { "X-API-Key": this.apiKey },
|
|
71
|
-
socketFactory: this.socketFactory ?? await defaultSocketFactory(),
|
|
72
|
-
retry: this.retryConfig,
|
|
73
|
-
replayBufferSize: (config["replay_buffer_size"] as number) ?? 32,
|
|
74
|
-
onReplay: (event, count) => {
|
|
75
|
-
this.emitMetric("", `tts.cartesia.reconnect_replay_${event}`, String(count));
|
|
76
|
-
},
|
|
77
|
-
keepAliveIntervalMs: KEEP_ALIVE_INTERVAL_MS,
|
|
78
|
-
onMessage: (data) => this.handleProviderMessage(data),
|
|
79
|
-
onConnectionLost: (err) => this.failActiveContexts(err),
|
|
80
|
-
onUnrecoverable: (err) => this.failActiveContexts(err),
|
|
81
|
-
});
|
|
82
|
-
await this.conn.connect();
|
|
46
|
+
class CartesiaWireProtocol implements WireProtocol {
|
|
47
|
+
constructor(private readonly cfg: CartesiaWireConfig) {}
|
|
83
48
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
const textPkt = pkt as { text: string; contextId: string };
|
|
87
|
-
await this.sendText(textPkt.text, textPkt.contextId);
|
|
88
|
-
}),
|
|
89
|
-
bus.on("tts.done", async (pkt: unknown) => {
|
|
90
|
-
const donePkt = pkt as { contextId: string };
|
|
91
|
-
if (!this.activeContexts.has(donePkt.contextId)) {
|
|
92
|
-
this.emitEnd(donePkt.contextId);
|
|
93
|
-
return;
|
|
94
|
-
}
|
|
95
|
-
if (finishTimeoutMs > 0) this.scheduleFinishTimeout(donePkt.contextId, finishTimeoutMs);
|
|
96
|
-
await this.finishContext(donePkt.contextId);
|
|
97
|
-
}),
|
|
98
|
-
bus.on("interrupt.tts", () => {
|
|
99
|
-
this.cancelActiveContexts().catch(() => {
|
|
100
|
-
// Best-effort interruption.
|
|
101
|
-
});
|
|
102
|
-
}),
|
|
103
|
-
);
|
|
49
|
+
attributionFor(contextId: string): { key: AttributionKey; contextId: string } {
|
|
50
|
+
return { key: attributionKey(contextId), contextId };
|
|
104
51
|
}
|
|
105
52
|
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
if (this.cancelledContexts.has(contextId)) return;
|
|
109
|
-
this.activeContexts.add(contextId);
|
|
110
|
-
const sent = await this.trySend(
|
|
53
|
+
encodeText(key: AttributionKey, text: string): SocketData[] {
|
|
54
|
+
return [
|
|
111
55
|
JSON.stringify({
|
|
112
|
-
model_id: this.modelId,
|
|
56
|
+
model_id: this.cfg.modelId,
|
|
113
57
|
transcript: text,
|
|
114
|
-
voice: { mode: "id", id: this.voiceId },
|
|
115
|
-
output_format: {
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
sample_rate: this.sampleRate,
|
|
119
|
-
},
|
|
120
|
-
language: this.language,
|
|
121
|
-
context_id: contextId || crypto.randomUUID(),
|
|
58
|
+
voice: { mode: "id", id: this.cfg.voiceId },
|
|
59
|
+
output_format: { container: "raw", encoding: "pcm_s16le", sample_rate: this.cfg.sampleRate },
|
|
60
|
+
language: this.cfg.language,
|
|
61
|
+
context_id: key || crypto.randomUUID(),
|
|
122
62
|
continue: true,
|
|
123
63
|
add_timestamps: true,
|
|
124
64
|
}),
|
|
125
|
-
|
|
126
|
-
);
|
|
127
|
-
if (!sent) {
|
|
128
|
-
this.activeContexts.delete(contextId);
|
|
129
|
-
}
|
|
65
|
+
];
|
|
130
66
|
}
|
|
131
67
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
await this.cancelActiveContexts();
|
|
136
|
-
return;
|
|
137
|
-
}
|
|
138
|
-
await this.cancelContext(contextId);
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
async finishContext(contextId: string): Promise<void> {
|
|
142
|
-
if (this.cancelledContexts.has(contextId)) return;
|
|
143
|
-
const sent = await this.trySend(
|
|
68
|
+
encodeFinish(contextId: string, activeKeys: readonly AttributionKey[]): SocketData[] {
|
|
69
|
+
if (activeKeys.length === 0) return [];
|
|
70
|
+
return [
|
|
144
71
|
JSON.stringify({
|
|
145
|
-
model_id: this.modelId,
|
|
72
|
+
model_id: this.cfg.modelId,
|
|
146
73
|
transcript: "",
|
|
147
|
-
voice: { mode: "id", id: this.voiceId },
|
|
148
|
-
output_format: {
|
|
149
|
-
|
|
150
|
-
encoding: "pcm_s16le",
|
|
151
|
-
sample_rate: this.sampleRate,
|
|
152
|
-
},
|
|
153
|
-
language: this.language,
|
|
74
|
+
voice: { mode: "id", id: this.cfg.voiceId },
|
|
75
|
+
output_format: { container: "raw", encoding: "pcm_s16le", sample_rate: this.cfg.sampleRate },
|
|
76
|
+
language: this.cfg.language,
|
|
154
77
|
context_id: contextId,
|
|
155
78
|
continue: false,
|
|
156
79
|
flush: true,
|
|
157
80
|
}),
|
|
158
|
-
|
|
159
|
-
);
|
|
160
|
-
if (!sent) {
|
|
161
|
-
this.activeContexts.delete(contextId);
|
|
162
|
-
}
|
|
81
|
+
];
|
|
163
82
|
}
|
|
164
83
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
this.activeContexts.clear();
|
|
168
|
-
this.cancelledContexts.clear();
|
|
169
|
-
for (const timer of this.finishTimers.values()) clearTimeout(timer);
|
|
170
|
-
this.finishTimers.clear();
|
|
171
|
-
await this.conn?.close();
|
|
172
|
-
this.conn = null;
|
|
173
|
-
this.bus = null;
|
|
84
|
+
encodeCancel(key: AttributionKey): SocketData[] {
|
|
85
|
+
return [JSON.stringify({ context_id: key, cancel: true })];
|
|
174
86
|
}
|
|
175
87
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
try {
|
|
179
|
-
await this.conn?.ensureReady();
|
|
180
|
-
this.conn?.send(payload);
|
|
181
|
-
return true;
|
|
182
|
-
} catch (err) {
|
|
183
|
-
this.emitError(contextId, err instanceof Error ? err : new Error(String(err)));
|
|
184
|
-
return false;
|
|
185
|
-
}
|
|
88
|
+
encodeClose(): SocketData[] {
|
|
89
|
+
return [];
|
|
186
90
|
}
|
|
187
91
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
this.activeContexts.clear();
|
|
192
|
-
await Promise.all(contextIds.map((contextId) => this.cancelContext(contextId)));
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
private async cancelContext(contextId: string): Promise<void> {
|
|
196
|
-
if (!contextId) return;
|
|
197
|
-
this.cancelledContexts.add(contextId);
|
|
198
|
-
this.activeContexts.delete(contextId);
|
|
199
|
-
this.clearFinishTimeout(contextId);
|
|
200
|
-
await this.trySend(
|
|
201
|
-
JSON.stringify({
|
|
202
|
-
context_id: contextId,
|
|
203
|
-
cancel: true,
|
|
204
|
-
}),
|
|
205
|
-
contextId,
|
|
206
|
-
);
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
private handleProviderMessage(data: SocketData): void {
|
|
210
|
-
if (typeof data !== "string") return; // Cartesia frames are JSON text
|
|
211
|
-
let msg: Record<string, unknown>;
|
|
212
|
-
try {
|
|
213
|
-
msg = JSON.parse(data) as Record<string, unknown>;
|
|
214
|
-
} catch (err) {
|
|
215
|
-
this.failActiveContexts(err instanceof Error ? err : new Error(String(err)));
|
|
216
|
-
return;
|
|
217
|
-
}
|
|
218
|
-
|
|
92
|
+
decode(data: SocketData): WireEvent[] {
|
|
93
|
+
if (typeof data !== "string") return []; // Cartesia frames are JSON text
|
|
94
|
+
const msg = JSON.parse(data) as Record<string, unknown>; // parse failure → engine fails all contexts
|
|
219
95
|
const contextId = typeof msg["context_id"] === "string" ? msg["context_id"] : "";
|
|
220
|
-
|
|
221
|
-
if (msg["done"] === true || msg["type"] === "error" || isErrorStatusCode(msg["status_code"])) {
|
|
222
|
-
this.cancelledContexts.delete(contextId);
|
|
223
|
-
}
|
|
224
|
-
return;
|
|
225
|
-
}
|
|
96
|
+
const key = attributionKey(contextId);
|
|
226
97
|
|
|
227
98
|
if (msg["type"] === "error" || isErrorStatusCode(msg["status_code"])) {
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
this.emitError(contextId, cartesiaProviderError(msg));
|
|
231
|
-
if (msg["done"] === true) this.emitEnd(contextId);
|
|
232
|
-
return;
|
|
99
|
+
// A `done:true` error frame both reports the error AND ends the context.
|
|
100
|
+
return [{ type: "error", key: contextId ? key : null, error: cartesiaProviderError(msg), endsContext: msg["done"] === true }];
|
|
233
101
|
}
|
|
234
102
|
|
|
103
|
+
const events: WireEvent[] = [];
|
|
235
104
|
if (msg["type"] === "timestamps") {
|
|
236
|
-
|
|
105
|
+
const words = parseWordTimestamps(msg["word_timestamps"]);
|
|
106
|
+
if (contextId && words.length > 0) {
|
|
107
|
+
events.push({
|
|
108
|
+
type: "sideband",
|
|
109
|
+
key,
|
|
110
|
+
route: Route.Main,
|
|
111
|
+
build: (ctxId, timestampMs) =>
|
|
112
|
+
({ kind: "tts.word_timestamps", contextId: ctxId, timestampMs, words } satisfies TextToSpeechWordTimestampsPacket),
|
|
113
|
+
});
|
|
114
|
+
}
|
|
237
115
|
}
|
|
238
|
-
|
|
239
|
-
//
|
|
240
|
-
// `flush_done` (the acknowledgement of a `flush: true` request) carry an empty
|
|
241
|
-
// `data` string and must not be decoded as audio.
|
|
116
|
+
// Audio arrives as non-empty base64 `data`; control frames such as `flush_done` carry an
|
|
117
|
+
// empty `data` and must not be decoded as audio.
|
|
242
118
|
if (typeof msg["data"] === "string" && msg["data"].length > 0) {
|
|
243
119
|
try {
|
|
244
|
-
const
|
|
245
|
-
assertAudioPayload(this.audioFormat,
|
|
246
|
-
|
|
247
|
-
kind: "tts.audio",
|
|
248
|
-
contextId,
|
|
249
|
-
timestampMs: Date.now(),
|
|
250
|
-
audio: new Uint8Array(audioBytes),
|
|
251
|
-
sampleRateHz: this.sampleRate,
|
|
252
|
-
provider: { name: "cartesia", model: this.modelId, region: "global", cancelled: false },
|
|
253
|
-
};
|
|
254
|
-
this.bus?.push(Route.Main, audioPacket);
|
|
120
|
+
const bytes = new Uint8Array(decodeStrictBase64(msg["data"], "Cartesia TTS provider audio data"));
|
|
121
|
+
assertAudioPayload(this.cfg.audioFormat, bytes);
|
|
122
|
+
events.push({ type: "audio", key, pcm: bytes });
|
|
255
123
|
} catch (err) {
|
|
256
|
-
|
|
257
|
-
this.emitError(contextId, err instanceof Error ? err : new Error(String(err)));
|
|
124
|
+
events.push({ type: "error", key, error: err instanceof Error ? err : new Error(String(err)) });
|
|
258
125
|
}
|
|
259
126
|
}
|
|
260
|
-
if (msg["done"] === true) {
|
|
261
|
-
|
|
262
|
-
this.clearFinishTimeout(contextId);
|
|
263
|
-
this.emitEnd(contextId);
|
|
264
|
-
}
|
|
127
|
+
if (msg["done"] === true) events.push({ type: "context_end", key });
|
|
128
|
+
return events;
|
|
265
129
|
}
|
|
130
|
+
}
|
|
266
131
|
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
contextId,
|
|
272
|
-
timestampMs: Date.now(),
|
|
273
|
-
component: "tts" as const,
|
|
274
|
-
category,
|
|
275
|
-
cause: err,
|
|
276
|
-
isRecoverable: isRecoverable(category),
|
|
277
|
-
};
|
|
278
|
-
this.bus?.push(Route.Critical, packet);
|
|
279
|
-
}
|
|
132
|
+
export class CartesiaTTSPlugin implements VoicePlugin {
|
|
133
|
+
// socketFactory is injectable so the same plugin runs on Node (default) or
|
|
134
|
+
// Cloudflare Workers (pass createWorkersSocket).
|
|
135
|
+
constructor(private readonly socketFactory?: SocketFactory) {}
|
|
280
136
|
|
|
281
|
-
private
|
|
282
|
-
this.clearFinishTimeout(contextId);
|
|
283
|
-
const timer = setTimeout(() => {
|
|
284
|
-
this.finishTimers.delete(contextId);
|
|
285
|
-
if (!this.activeContexts.has(contextId)) return;
|
|
286
|
-
this.emitMetric(contextId, "tts.cartesia.finish_timeout", String(timeoutMs));
|
|
287
|
-
this.activeContexts.delete(contextId);
|
|
288
|
-
this.emitEnd(contextId);
|
|
289
|
-
}, timeoutMs);
|
|
290
|
-
this.finishTimers.set(contextId, timer);
|
|
291
|
-
}
|
|
137
|
+
private session: StreamingTtsSession | null = null;
|
|
292
138
|
|
|
293
|
-
|
|
294
|
-
const
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
139
|
+
async initialize(bus: PipelineBus, config: PluginConfig): Promise<void> {
|
|
140
|
+
const apiKey = requireStringConfig(config, "api_key");
|
|
141
|
+
const voiceId = optionalStringConfig(config, "voice_id") ?? "c2ac25f9-ecc4-4f56-9095-651354df60c0";
|
|
142
|
+
const modelId = optionalStringConfig(config, "model_id") ?? "sonic-3";
|
|
143
|
+
const endpointUrl = optionalStringConfig(config, "endpoint_url") ?? "wss://api.cartesia.ai/tts/websocket";
|
|
144
|
+
const apiVersion = optionalStringConfig(config, "cartesia_version") ?? "2024-06-10";
|
|
145
|
+
const sampleRate = (config["sample_rate"] as number) ?? 16000;
|
|
146
|
+
const language = optionalStringConfig(config, "language") ?? "en";
|
|
147
|
+
const audioFormat: AudioFormat = { encoding: "pcm_s16le", sampleRateHz: sampleRate, channels: 1 };
|
|
148
|
+
assertAudioFormat(audioFormat);
|
|
149
|
+
|
|
150
|
+
this.session = await startStreamingTtsSession(bus, {
|
|
151
|
+
protocol: new CartesiaWireProtocol({ modelId, voiceId, sampleRate, language, audioFormat }),
|
|
152
|
+
provider: { name: "cartesia", model: modelId, region: "global" },
|
|
153
|
+
format: audioFormat,
|
|
154
|
+
sampleRateHz: sampleRate,
|
|
155
|
+
url: () => {
|
|
156
|
+
const params = new URLSearchParams({ cartesia_version: apiVersion });
|
|
157
|
+
const separator = endpointUrl.includes("?") ? "&" : "?";
|
|
158
|
+
return `${endpointUrl}${separator}${params.toString()}`;
|
|
159
|
+
},
|
|
160
|
+
headers: { "X-API-Key": apiKey },
|
|
161
|
+
retry: readProviderRetryConfig(config),
|
|
162
|
+
finishTimeoutMs: readNonNegativeInteger(config["finish_timeout_ms"], 2000),
|
|
163
|
+
metricPrefix: "tts.cartesia",
|
|
164
|
+
socketFactory: this.socketFactory ?? (await defaultNodeSocketFactory()),
|
|
165
|
+
replayBufferSize: (config["replay_buffer_size"] as number) ?? 32,
|
|
166
|
+
keepAliveIntervalMs: KEEP_ALIVE_INTERVAL_MS,
|
|
307
167
|
});
|
|
308
168
|
}
|
|
309
169
|
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
this.
|
|
313
|
-
if (contextIds.length === 0) {
|
|
314
|
-
this.emitError("", err);
|
|
315
|
-
return;
|
|
316
|
-
}
|
|
317
|
-
for (const contextId of contextIds) {
|
|
318
|
-
this.emitError(contextId, err);
|
|
319
|
-
}
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
private emitEnd(contextId: string): void {
|
|
323
|
-
const packet: TextToSpeechEndPacket = {
|
|
324
|
-
kind: "tts.end",
|
|
325
|
-
contextId,
|
|
326
|
-
timestampMs: Date.now(),
|
|
327
|
-
};
|
|
328
|
-
this.bus?.push(Route.Main, packet);
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
private emitWordTimestamps(contextId: string, value: unknown): void {
|
|
332
|
-
if (!contextId || value === null || typeof value !== "object") return;
|
|
333
|
-
const raw = value as Record<string, unknown>;
|
|
334
|
-
const rawWords = Array.isArray(raw["words"]) ? raw["words"] : [];
|
|
335
|
-
const rawStarts = Array.isArray(raw["start"]) ? raw["start"] : [];
|
|
336
|
-
const rawEnds = Array.isArray(raw["end"]) ? raw["end"] : [];
|
|
337
|
-
const count = Math.min(rawWords.length, rawStarts.length, rawEnds.length);
|
|
338
|
-
const words: TtsWordTimestamp[] = [];
|
|
339
|
-
for (let i = 0; i < count; i += 1) {
|
|
340
|
-
const word = rawWords[i];
|
|
341
|
-
const start = rawStarts[i];
|
|
342
|
-
const end = rawEnds[i];
|
|
343
|
-
if (typeof word !== "string" || typeof start !== "number" || typeof end !== "number") continue;
|
|
344
|
-
words.push({
|
|
345
|
-
word,
|
|
346
|
-
startMs: Math.round(start * 1000),
|
|
347
|
-
endMs: Math.round(end * 1000),
|
|
348
|
-
});
|
|
349
|
-
}
|
|
350
|
-
if (words.length === 0) return;
|
|
351
|
-
this.bus?.push(Route.Main, {
|
|
352
|
-
kind: "tts.word_timestamps",
|
|
353
|
-
contextId,
|
|
354
|
-
timestampMs: Date.now(),
|
|
355
|
-
words,
|
|
356
|
-
});
|
|
170
|
+
async close(): Promise<void> {
|
|
171
|
+
await this.session?.dispose();
|
|
172
|
+
this.session = null;
|
|
357
173
|
}
|
|
358
174
|
}
|
|
359
175
|
|
|
360
|
-
async function defaultSocketFactory(): Promise<SocketFactory> {
|
|
361
|
-
const mod = await import("@kuralle-syrinx/ws/node");
|
|
362
|
-
return mod.createNodeWsSocket;
|
|
363
|
-
}
|
|
364
|
-
|
|
365
176
|
function isErrorStatusCode(value: unknown): boolean {
|
|
366
177
|
return typeof value === "number" && value >= 400;
|
|
367
178
|
}
|
|
@@ -388,6 +199,24 @@ function decodeStrictBase64(value: string, name: string): Buffer {
|
|
|
388
199
|
return decoded;
|
|
389
200
|
}
|
|
390
201
|
|
|
202
|
+
function parseWordTimestamps(value: unknown): TtsWordTimestamp[] {
|
|
203
|
+
if (value === null || typeof value !== "object") return [];
|
|
204
|
+
const raw = value as Record<string, unknown>;
|
|
205
|
+
const rawWords = Array.isArray(raw["words"]) ? raw["words"] : [];
|
|
206
|
+
const rawStarts = Array.isArray(raw["start"]) ? raw["start"] : [];
|
|
207
|
+
const rawEnds = Array.isArray(raw["end"]) ? raw["end"] : [];
|
|
208
|
+
const count = Math.min(rawWords.length, rawStarts.length, rawEnds.length);
|
|
209
|
+
const words: TtsWordTimestamp[] = [];
|
|
210
|
+
for (let i = 0; i < count; i += 1) {
|
|
211
|
+
const word = rawWords[i];
|
|
212
|
+
const start = rawStarts[i];
|
|
213
|
+
const end = rawEnds[i];
|
|
214
|
+
if (typeof word !== "string" || typeof start !== "number" || typeof end !== "number") continue;
|
|
215
|
+
words.push({ word, startMs: Math.round(start * 1000), endMs: Math.round(end * 1000) });
|
|
216
|
+
}
|
|
217
|
+
return words;
|
|
218
|
+
}
|
|
219
|
+
|
|
391
220
|
function readNonNegativeInteger(value: unknown, fallback: number): number {
|
|
392
221
|
if (typeof value !== "number" || !Number.isFinite(value)) return fallback;
|
|
393
222
|
const integer = Math.floor(value);
|