@micdrop/pocket-tts 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md ADDED
@@ -0,0 +1,83 @@
1
+ # @micdrop/pocket-tts
2
+
3
+ [Micdrop website](https://micdrop.dev) | [Documentation](https://micdrop.dev/docs/ai-integration/provided-integrations/pocket-tts)
4
+
5
+ Local [Pocket TTS](https://github.com/kyutai-labs/pocket-tts) text to speech for
6
+ [@micdrop/server](https://micdrop.dev/docs/server).
7
+
8
+ Runs in your Node process through the sherpa-onnx addon and ONNX Runtime, so
9
+ there is no API key, no Python next to your server, and no text leaving the
10
+ machine. Pocket TTS is a 100 million parameter model from Kyutai, built for the
11
+ CPU, that clones the voice of a few seconds of reference audio and hands its
12
+ audio over while it is still generating the rest.
13
+
14
+ It speaks English. Use [@micdrop/piper](https://micdrop.dev/docs/ai-integration/provided-integrations/piper)
15
+ for the other languages.
16
+
17
+ ## Installation
18
+
19
+ Install the package:
20
+
21
+ ```bash
22
+ npm install @micdrop/pocket-tts
23
+ ```
24
+
25
+ Download the weights, converted to ONNX by the sherpa-onnx project:
26
+
27
+ ```bash
28
+ curl -LO https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2
29
+ tar xf sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2
30
+ rm sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2
31
+ ```
32
+
33
+ The archive weighs 98 MB and takes 190 MB on disk. A full precision archive,
34
+ `sherpa-onnx-pocket-tts-2026-01-26.tar.bz2`, exists next to it and is read the
35
+ same way.
36
+
37
+ ## Usage
38
+
39
+ ```typescript
40
+ import { PocketTTS } from '@micdrop/pocket-tts'
41
+ import { MicdropServer } from '@micdrop/server'
42
+
43
+ const tts = new PocketTTS({
44
+ modelDir: './sherpa-onnx-pocket-tts-int8-2026-01-26',
45
+ voice: 'bria', // or the path to a wav of the voice to clone
46
+ })
47
+
48
+ // Use with MicdropServer
49
+ new MicdropServer(socket, {
50
+ tts,
51
+ // ... other options
52
+ })
53
+ ```
54
+
55
+ ## Voices
56
+
57
+ The voice comes from a reference recording rather than from a catalog: a few
58
+ seconds of someone speaking is enough for the model to answer in that voice.
59
+ The archive ships three samples in its `test_wavs` folder, reachable by name
60
+ through `BUNDLED_VOICES`, and any wav file works in their place.
61
+
62
+ ```typescript
63
+ const tts = new PocketTTS({
64
+ modelDir: './sherpa-onnx-pocket-tts-int8-2026-01-26',
65
+ voice: './voices/my-voice.wav',
66
+ })
67
+ ```
68
+
69
+ Clone a voice only with the consent of the person it belongs to, which the
70
+ [Kyutai license](https://github.com/kyutai-labs/pocket-tts#prohibited-use)
71
+ requires.
72
+
73
+ ## Documentation
74
+
75
+ Read full [documentation of the Pocket TTS integration for Micdrop](https://micdrop.dev/docs/ai-integration/provided-integrations/pocket-tts) on the [website](https://micdrop.dev), and the [guide on running Micdrop with local models](https://micdrop.dev/docs/ai-integration/local-models).
76
+
77
+ ## License
78
+
79
+ MIT
80
+
81
+ ## Author
82
+
83
+ Originally developed for [Raconte.ai](https://www.raconte.ai), created and open sourced by [Godefroy de Compreignac](https://github.com/Godefroy)
@@ -0,0 +1,76 @@
1
+ import { PocketModelConfig } from 'sherpa-onnx-node';
2
+ import { SentenceTTS } from '@micdrop/server';
3
+
4
+ declare function resolveModelFiles(modelDir: string): Omit<PocketModelConfig, 'voiceEmbeddingCacheCapacity'>;
5
+ /**
6
+ * The reference voices the archive ships in its `test_wavs` folder.
7
+ *
8
+ * `bria` is a woman reading calmly for forty seconds, `loona` is a one second
9
+ * clip, which is the shortest a reference can reasonably be, and
10
+ * `frenchAccent` is a French speaker, cloned into English words: the language
11
+ * comes from the text, the timbre and the accent from the recording.
12
+ */
13
+ declare const BUNDLED_VOICES: {
14
+ readonly bria: "test_wavs/bria.wav";
15
+ readonly loona: "test_wavs/loona.wav";
16
+ readonly frenchAccent: "test_wavs/sample_fr_hibiki_crepes.wav";
17
+ };
18
+ type PocketBundledVoice = keyof typeof BUNDLED_VOICES;
19
+ /** Turns a bundled voice name into a path, and leaves a path as it is. */
20
+ declare function resolveVoicePath(modelDir: string, voice: PocketBundledVoice | (string & {})): string;
21
+
22
+ /**
23
+ * Local Pocket TTS text to speech, running in the Node process through the
24
+ * sherpa-onnx addon and ONNX Runtime. Nothing leaves the machine and no server
25
+ * has to be started next to it.
26
+ *
27
+ * Pocket TTS is a 100 million parameter model from Kyutai, built for the CPU,
28
+ * that clones the voice of a few seconds of reference audio. It generates
29
+ * several times faster than real time and hands its audio over while it is
30
+ * still writing the rest, so the first words of a sentence reach the client in
31
+ * about three hundred milliseconds.
32
+ *
33
+ * The weights are downloaded separately, see the README.
34
+ *
35
+ * @see https://github.com/kyutai-labs/pocket-tts
36
+ * @see https://k2-fsa.github.io/sherpa/onnx/tts/pocket.html
37
+ */
38
+ interface PocketTTSOptions {
39
+ /** Folder holding the model files, as extracted from the archive. */
40
+ modelDir: string;
41
+ /**
42
+ * Voice to clone: a bundled name ("bria"), or the path to a wav file
43
+ * carrying a few seconds of the voice to speak with.
44
+ */
45
+ voice?: PocketBundledVoice | (string & {});
46
+ /** Speech rate, 1 being the natural pace of the voice. */
47
+ speed?: number;
48
+ /** Sampling steps per frame, higher sounding better and generating slower. */
49
+ numSteps?: number;
50
+ /** Seconds of the reference audio the voice is built from. */
51
+ referenceLength?: number;
52
+ /** Sampling seed, so the same sentence comes out the same way. */
53
+ seed?: number;
54
+ /** Threads given to ONNX Runtime, 2 by default. */
55
+ numThreads?: number;
56
+ /** Execution provider passed to ONNX Runtime, "cpu" by default. */
57
+ provider?: 'cpu' | 'cuda' | 'coreml';
58
+ /** Synthesizes one word at startup, on by default. */
59
+ warmup?: boolean;
60
+ }
61
+ declare class PocketTTS extends SentenceTTS {
62
+ private readonly options;
63
+ constructor(options: PocketTTSOptions);
64
+ protected synthesize(text: string, signal: AbortSignal): Promise<undefined>;
65
+ /**
66
+ * Speaks one short word to pay the first inference up front.
67
+ *
68
+ * It also builds the embedding of the voice, which the addon then keeps, so
69
+ * the first sentence of the call only pays for its own generation.
70
+ */
71
+ private warmup;
72
+ private getGenerationConfig;
73
+ private getSynthesizerOptions;
74
+ }
75
+
76
+ export { BUNDLED_VOICES, type PocketBundledVoice, PocketTTS, type PocketTTSOptions, resolveModelFiles, resolveVoicePath };
@@ -0,0 +1,76 @@
1
+ import { PocketModelConfig } from 'sherpa-onnx-node';
2
+ import { SentenceTTS } from '@micdrop/server';
3
+
4
+ declare function resolveModelFiles(modelDir: string): Omit<PocketModelConfig, 'voiceEmbeddingCacheCapacity'>;
5
+ /**
6
+ * The reference voices the archive ships in its `test_wavs` folder.
7
+ *
8
+ * `bria` is a woman reading calmly for forty seconds, `loona` is a one second
9
+ * clip, which is the shortest a reference can reasonably be, and
10
+ * `frenchAccent` is a French speaker, cloned into English words: the language
11
+ * comes from the text, the timbre and the accent from the recording.
12
+ */
13
+ declare const BUNDLED_VOICES: {
14
+ readonly bria: "test_wavs/bria.wav";
15
+ readonly loona: "test_wavs/loona.wav";
16
+ readonly frenchAccent: "test_wavs/sample_fr_hibiki_crepes.wav";
17
+ };
18
+ type PocketBundledVoice = keyof typeof BUNDLED_VOICES;
19
+ /** Turns a bundled voice name into a path, and leaves a path as it is. */
20
+ declare function resolveVoicePath(modelDir: string, voice: PocketBundledVoice | (string & {})): string;
21
+
22
+ /**
23
+ * Local Pocket TTS text to speech, running in the Node process through the
24
+ * sherpa-onnx addon and ONNX Runtime. Nothing leaves the machine and no server
25
+ * has to be started next to it.
26
+ *
27
+ * Pocket TTS is a 100 million parameter model from Kyutai, built for the CPU,
28
+ * that clones the voice of a few seconds of reference audio. It generates
29
+ * several times faster than real time and hands its audio over while it is
30
+ * still writing the rest, so the first words of a sentence reach the client in
31
+ * about three hundred milliseconds.
32
+ *
33
+ * The weights are downloaded separately, see the README.
34
+ *
35
+ * @see https://github.com/kyutai-labs/pocket-tts
36
+ * @see https://k2-fsa.github.io/sherpa/onnx/tts/pocket.html
37
+ */
38
+ interface PocketTTSOptions {
39
+ /** Folder holding the model files, as extracted from the archive. */
40
+ modelDir: string;
41
+ /**
42
+ * Voice to clone: a bundled name ("bria"), or the path to a wav file
43
+ * carrying a few seconds of the voice to speak with.
44
+ */
45
+ voice?: PocketBundledVoice | (string & {});
46
+ /** Speech rate, 1 being the natural pace of the voice. */
47
+ speed?: number;
48
+ /** Sampling steps per frame, higher sounding better and generating slower. */
49
+ numSteps?: number;
50
+ /** Seconds of the reference audio the voice is built from. */
51
+ referenceLength?: number;
52
+ /** Sampling seed, so the same sentence comes out the same way. */
53
+ seed?: number;
54
+ /** Threads given to ONNX Runtime, 2 by default. */
55
+ numThreads?: number;
56
+ /** Execution provider passed to ONNX Runtime, "cpu" by default. */
57
+ provider?: 'cpu' | 'cuda' | 'coreml';
58
+ /** Synthesizes one word at startup, on by default. */
59
+ warmup?: boolean;
60
+ }
61
+ declare class PocketTTS extends SentenceTTS {
62
+ private readonly options;
63
+ constructor(options: PocketTTSOptions);
64
+ protected synthesize(text: string, signal: AbortSignal): Promise<undefined>;
65
+ /**
66
+ * Speaks one short word to pay the first inference up front.
67
+ *
68
+ * It also builds the embedding of the voice, which the addon then keeps, so
69
+ * the first sentence of the call only pays for its own generation.
70
+ */
71
+ private warmup;
72
+ private getGenerationConfig;
73
+ private getSynthesizerOptions;
74
+ }
75
+
76
+ export { BUNDLED_VOICES, type PocketBundledVoice, PocketTTS, type PocketTTSOptions, resolveModelFiles, resolveVoicePath };
package/dist/index.js ADDED
@@ -0,0 +1,219 @@
1
+ "use strict";
2
+ var __create = Object.create;
3
+ var __defProp = Object.defineProperty;
4
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
+ var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
7
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
8
+ var __export = (target, all) => {
9
+ for (var name in all)
10
+ __defProp(target, name, { get: all[name], enumerable: true });
11
+ };
12
+ var __copyProps = (to, from, except, desc) => {
13
+ if (from && typeof from === "object" || typeof from === "function") {
14
+ for (let key of __getOwnPropNames(from))
15
+ if (!__hasOwnProp.call(to, key) && key !== except)
16
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
+ }
18
+ return to;
19
+ };
20
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
+ // If the importer is in node compatibility mode or this is not an ESM
22
+ // file that has been converted to a CommonJS file using a Babel-
23
+ // compatible transform (i.e. "__esModule" has not been set), then set
24
+ // "default" to the CommonJS "module.exports" for node compatibility.
25
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
+ mod
27
+ ));
28
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
+
30
+ // src/index.ts
31
+ var index_exports = {};
32
+ __export(index_exports, {
33
+ BUNDLED_VOICES: () => BUNDLED_VOICES,
34
+ PocketTTS: () => PocketTTS,
35
+ resolveModelFiles: () => resolveModelFiles,
36
+ resolveVoicePath: () => resolveVoicePath
37
+ });
38
+ module.exports = __toCommonJS(index_exports);
39
+
40
+ // src/modelFiles.ts
41
+ var import_fs = require("fs");
42
+ var import_path = require("path");
43
+ var WEIGHTS = {
44
+ lmFlow: "lm_flow",
45
+ lmMain: "lm_main",
46
+ encoder: "encoder",
47
+ decoder: "decoder",
48
+ textConditioner: "text_conditioner"
49
+ };
50
+ var JSON_FILES = {
51
+ vocabJson: "vocab.json",
52
+ tokenScoresJson: "token_scores.json"
53
+ };
54
+ function resolveModelFiles(modelDir) {
55
+ const files = {};
56
+ const missing = [];
57
+ for (const [key, base] of Object.entries(WEIGHTS)) {
58
+ const found = [`${base}.int8.onnx`, `${base}.onnx`].map((name) => (0, import_path.join)(modelDir, name)).find(import_fs.existsSync);
59
+ if (found) files[key] = found;
60
+ else missing.push(`${base}.int8.onnx or ${base}.onnx`);
61
+ }
62
+ for (const [key, name] of Object.entries(JSON_FILES)) {
63
+ const path = (0, import_path.join)(modelDir, name);
64
+ if ((0, import_fs.existsSync)(path)) files[key] = path;
65
+ else missing.push(name);
66
+ }
67
+ if (missing.length > 0) {
68
+ throw new Error(
69
+ `Pocket TTS model files not found in "${modelDir}": ${missing.join(", ")}. Download the archive from https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models`
70
+ );
71
+ }
72
+ return files;
73
+ }
74
+ var BUNDLED_VOICES = {
75
+ bria: "test_wavs/bria.wav",
76
+ loona: "test_wavs/loona.wav",
77
+ frenchAccent: "test_wavs/sample_fr_hibiki_crepes.wav"
78
+ };
79
+ function resolveVoicePath(modelDir, voice) {
80
+ const bundled = BUNDLED_VOICES[voice];
81
+ return bundled ? (0, import_path.join)(modelDir, bundled) : voice;
82
+ }
83
+
84
+ // src/PocketTTS.ts
85
+ var import_server = require("@micdrop/server");
86
+ var import_sherpa_onnx_node2 = __toESM(require("sherpa-onnx-node"));
87
+
88
+ // src/synthesizer.ts
89
+ var import_fs2 = require("fs");
90
+ var import_sherpa_onnx_node = __toESM(require("sherpa-onnx-node"));
91
+ var DEFAULT_NUM_THREADS = 2;
92
+ var DEFAULT_PROVIDER = "cpu";
93
+ var VOICE_CACHE_CAPACITY = 10;
94
+ var synthesizers = /* @__PURE__ */ new Map();
95
+ function cacheKey(options) {
96
+ return JSON.stringify([
97
+ options.modelDir,
98
+ options.numThreads ?? null,
99
+ options.provider ?? null,
100
+ options.debug ?? null
101
+ ]);
102
+ }
103
+ function loadSynthesizer(options) {
104
+ const key = cacheKey(options);
105
+ const existing = synthesizers.get(key);
106
+ if (existing) return existing;
107
+ const loading = Promise.resolve().then(
108
+ () => import_sherpa_onnx_node.default.OfflineTts.createAsync({
109
+ model: {
110
+ pocket: {
111
+ ...resolveModelFiles(options.modelDir),
112
+ voiceEmbeddingCacheCapacity: VOICE_CACHE_CAPACITY
113
+ },
114
+ numThreads: options.numThreads ?? DEFAULT_NUM_THREADS,
115
+ provider: options.provider ?? DEFAULT_PROVIDER,
116
+ debug: options.debug ?? false
117
+ },
118
+ // Sentences are already split upstream, one per synthesis
119
+ maxNumSentences: 1
120
+ })
121
+ );
122
+ loading.catch(() => synthesizers.delete(key));
123
+ synthesizers.set(key, loading);
124
+ return loading;
125
+ }
126
+ var voices = /* @__PURE__ */ new Map();
127
+ function loadVoice(path) {
128
+ const existing = voices.get(path);
129
+ if (existing) return existing;
130
+ if (!(0, import_fs2.existsSync)(path)) {
131
+ throw new Error(`Pocket TTS reference voice not found: "${path}"`);
132
+ }
133
+ const wave = import_sherpa_onnx_node.default.readWave(path);
134
+ voices.set(path, wave);
135
+ return wave;
136
+ }
137
+
138
+ // src/PocketTTS.ts
139
+ var DEFAULT_VOICE = "bria";
140
+ var DEFAULT_NUM_STEPS = 2;
141
+ var DEFAULT_REFERENCE_LENGTH = 12;
142
+ var OUTPUT_SAMPLE_RATE = 16e3;
143
+ var PocketTTS = class extends import_server.SentenceTTS {
144
+ constructor(options) {
145
+ super();
146
+ this.options = options;
147
+ resolveModelFiles(options.modelDir);
148
+ loadSynthesizer(this.getSynthesizerOptions()).catch((error) => {
149
+ console.error("[PocketTTS] Failed to load model:", error);
150
+ });
151
+ if (options.warmup !== false) {
152
+ this.warmup().then(() => this.log("Model warmed up")).catch(() => {
153
+ });
154
+ }
155
+ }
156
+ async synthesize(text, signal) {
157
+ const synthesizer = await loadSynthesizer(this.getSynthesizerOptions());
158
+ if (signal.aborted) return;
159
+ const resampler = new import_server.Pcm16Resampler(
160
+ synthesizer.sampleRate,
161
+ OUTPUT_SAMPLE_RATE
162
+ );
163
+ await synthesizer.generateAsync({
164
+ text,
165
+ // The samples of a chunk live in native memory for the length of the
166
+ // callback, which is where they are copied into a PCM16 buffer
167
+ enableExternalBuffer: true,
168
+ generationConfig: this.getGenerationConfig(),
169
+ onProgress: ({ samples }) => {
170
+ const audio = resampler.process((0, import_server.float32ToPcm16)(samples));
171
+ return this.emitAudio(audio) ? 1 : 0;
172
+ }
173
+ });
174
+ }
175
+ /**
176
+ * Speaks one short word to pay the first inference up front.
177
+ *
178
+ * It also builds the embedding of the voice, which the addon then keeps, so
179
+ * the first sentence of the call only pays for its own generation.
180
+ */
181
+ async warmup() {
182
+ const synthesizer = await loadSynthesizer(this.getSynthesizerOptions());
183
+ await synthesizer.generateAsync({
184
+ text: "Hello.",
185
+ generationConfig: this.getGenerationConfig()
186
+ });
187
+ }
188
+ getGenerationConfig() {
189
+ const { options } = this;
190
+ const voice = loadVoice(
191
+ resolveVoicePath(options.modelDir, options.voice ?? DEFAULT_VOICE)
192
+ );
193
+ return new import_sherpa_onnx_node2.default.GenerationConfig({
194
+ speed: options.speed,
195
+ numSteps: options.numSteps ?? DEFAULT_NUM_STEPS,
196
+ referenceAudio: voice.samples,
197
+ referenceSampleRate: voice.sampleRate,
198
+ extra: {
199
+ max_reference_audio_len: options.referenceLength ?? DEFAULT_REFERENCE_LENGTH,
200
+ ...options.seed === void 0 ? {} : { seed: options.seed }
201
+ }
202
+ });
203
+ }
204
+ getSynthesizerOptions() {
205
+ return {
206
+ modelDir: this.options.modelDir,
207
+ numThreads: this.options.numThreads,
208
+ provider: this.options.provider
209
+ };
210
+ }
211
+ };
212
+ // Annotate the CommonJS export names for ESM import in node:
213
+ 0 && (module.exports = {
214
+ BUNDLED_VOICES,
215
+ PocketTTS,
216
+ resolveModelFiles,
217
+ resolveVoicePath
218
+ });
219
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/index.ts","../src/modelFiles.ts","../src/PocketTTS.ts","../src/synthesizer.ts"],"sourcesContent":["export * from './modelFiles'\nexport * from './PocketTTS'\n","import { existsSync } from 'fs'\nimport { join } from 'path'\nimport type { PocketModelConfig } from 'sherpa-onnx-node'\n\n/**\n * The files a Pocket TTS archive holds, resolved inside its folder.\n *\n * The model ships in two flavours, quantized and full precision, which name\n * their weights `lm_main.int8.onnx` and `lm_main.onnx`. Each part is looked up\n * in that order, so both archives work and a folder mixing them takes the\n * lighter file whenever there is one, which is what the quantized archive\n * itself does for its encoder and its text conditioner.\n */\n\nconst WEIGHTS = {\n lmFlow: 'lm_flow',\n lmMain: 'lm_main',\n encoder: 'encoder',\n decoder: 'decoder',\n textConditioner: 'text_conditioner',\n} as const\n\nconst JSON_FILES = {\n vocabJson: 'vocab.json',\n tokenScoresJson: 'token_scores.json',\n} as const\n\nexport function resolveModelFiles(\n modelDir: string\n): Omit<PocketModelConfig, 'voiceEmbeddingCacheCapacity'> {\n const files: Record<string, string> = {}\n const missing: string[] = []\n\n for (const [key, base] of Object.entries(WEIGHTS)) {\n const found = [`${base}.int8.onnx`, `${base}.onnx`]\n .map((name) => join(modelDir, name))\n .find(existsSync)\n if (found) files[key] = found\n else missing.push(`${base}.int8.onnx or ${base}.onnx`)\n }\n\n for (const [key, name] of Object.entries(JSON_FILES)) {\n const path = join(modelDir, name)\n if (existsSync(path)) files[key] = path\n else missing.push(name)\n }\n\n if (missing.length > 0) {\n throw new Error(\n `Pocket TTS model files not found in \"${modelDir}\": ${missing.join(', ')}. ` +\n 'Download the archive from https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models'\n )\n }\n\n return files as Omit<PocketModelConfig, 'voiceEmbeddingCacheCapacity'>\n}\n\n/**\n * The reference voices the archive ships in its `test_wavs` folder.\n *\n * `bria` is a woman reading calmly for forty seconds, `loona` is a one second\n * clip, which is the shortest a reference can reasonably be, and\n * `frenchAccent` is a French speaker, cloned into English words: the language\n * comes from the text, the timbre and the accent from the recording.\n */\nexport const BUNDLED_VOICES = {\n bria: 'test_wavs/bria.wav',\n loona: 'test_wavs/loona.wav',\n frenchAccent: 'test_wavs/sample_fr_hibiki_crepes.wav',\n} as const\n\nexport type PocketBundledVoice = keyof typeof BUNDLED_VOICES\n\n/** Turns a bundled voice name into a path, and leaves a path as it is. */\nexport function resolveVoicePath(\n modelDir: string,\n voice: PocketBundledVoice | (string & {})\n): string {\n const bundled = BUNDLED_VOICES[voice as PocketBundledVoice]\n return bundled ? join(modelDir, bundled) : voice\n}\n","import { float32ToPcm16, Pcm16Resampler, SentenceTTS } from '@micdrop/server'\nimport sherpa, { type GenerationConfig } from 'sherpa-onnx-node'\nimport {\n PocketBundledVoice,\n resolveModelFiles,\n resolveVoicePath,\n} from './modelFiles'\nimport { loadSynthesizer, loadVoice, SynthesizerOptions } from './synthesizer'\n\n/**\n * Local Pocket TTS text to speech, running in the Node process through the\n * sherpa-onnx addon and ONNX Runtime. Nothing leaves the machine and no server\n * has to be started next to it.\n *\n * Pocket TTS is a 100 million parameter model from Kyutai, built for the CPU,\n * that clones the voice of a few seconds of reference audio. It generates\n * several times faster than real time and hands its audio over while it is\n * still writing the rest, so the first words of a sentence reach the client in\n * about three hundred milliseconds.\n *\n * The weights are downloaded separately, see the README.\n *\n * @see https://github.com/kyutai-labs/pocket-tts\n * @see https://k2-fsa.github.io/sherpa/onnx/tts/pocket.html\n */\n\nexport interface PocketTTSOptions {\n /** Folder holding the model files, as extracted from the archive. */\n modelDir: string\n\n /**\n * Voice to clone: a bundled name (\"bria\"), or the path to a wav file\n * carrying a few seconds of the voice to speak with.\n */\n voice?: PocketBundledVoice | (string & {})\n\n /** Speech rate, 1 being the natural pace of the voice. */\n speed?: number\n\n /** Sampling steps per frame, higher sounding better and generating slower. */\n numSteps?: number\n\n /** Seconds of the reference audio the voice is built from. */\n referenceLength?: number\n\n /** Sampling seed, so the same sentence comes out the same way. */\n seed?: number\n\n /** Threads given to ONNX Runtime, 2 by default. */\n numThreads?: number\n\n /** Execution provider passed to ONNX Runtime, \"cpu\" by default. */\n provider?: 'cpu' | 'cuda' | 'coreml'\n\n /** Synthesizes one word at startup, on by default. */\n warmup?: boolean\n}\n\nconst DEFAULT_VOICE: PocketBundledVoice = 'bria'\nconst DEFAULT_NUM_STEPS = 2\nconst DEFAULT_REFERENCE_LENGTH = 12\nconst OUTPUT_SAMPLE_RATE = 16000 // Rate expected by the Micdrop client\n\nexport class PocketTTS extends SentenceTTS {\n constructor(private readonly options: PocketTTSOptions) {\n super()\n\n // Fail at startup rather than on the first sentence of the call\n resolveModelFiles(options.modelDir)\n\n loadSynthesizer(this.getSynthesizerOptions()).catch((error) => {\n console.error('[PocketTTS] Failed to load model:', error)\n })\n\n if (options.warmup !== false) {\n this.warmup()\n .then(() => this.log('Model warmed up'))\n .catch(() => {})\n }\n }\n\n protected async synthesize(\n text: string,\n signal: AbortSignal\n ): Promise<undefined> {\n const synthesizer = await loadSynthesizer(this.getSynthesizerOptions())\n // The utterance may have been cancelled while the model was loading\n if (signal.aborted) return\n\n // One resampler for the whole sentence: its chunks are a single continuous\n // stream, and the resampler carries its fractional position across them\n const resampler = new Pcm16Resampler(\n synthesizer.sampleRate,\n OUTPUT_SAMPLE_RATE\n )\n\n await synthesizer.generateAsync({\n text,\n // The samples of a chunk live in native memory for the length of the\n // callback, which is where they are copied into a PCM16 buffer\n enableExternalBuffer: true,\n generationConfig: this.getGenerationConfig(),\n onProgress: ({ samples }) => {\n const audio = resampler.process(float32ToPcm16(samples))\n // Stop generating what nobody is going to hear\n return this.emitAudio(audio) ? 1 : 0\n },\n })\n }\n\n /**\n * Speaks one short word to pay the first inference up front.\n *\n * It also builds the embedding of the voice, which the addon then keeps, so\n * the first sentence of the call only pays for its own generation.\n */\n private async warmup(): Promise<void> {\n const synthesizer = await loadSynthesizer(this.getSynthesizerOptions())\n await synthesizer.generateAsync({\n text: 'Hello.',\n generationConfig: this.getGenerationConfig(),\n })\n }\n\n private getGenerationConfig(): GenerationConfig {\n const { options } = this\n const voice = loadVoice(\n resolveVoicePath(options.modelDir, options.voice ?? DEFAULT_VOICE)\n )\n\n return new sherpa.GenerationConfig({\n speed: options.speed,\n numSteps: options.numSteps ?? DEFAULT_NUM_STEPS,\n referenceAudio: voice.samples,\n referenceSampleRate: voice.sampleRate,\n extra: {\n max_reference_audio_len:\n options.referenceLength ?? DEFAULT_REFERENCE_LENGTH,\n ...(options.seed === undefined ? {} : { seed: options.seed }),\n },\n })\n }\n\n private getSynthesizerOptions(): SynthesizerOptions {\n return {\n modelDir: this.options.modelDir,\n numThreads: this.options.numThreads,\n provider: this.options.provider,\n }\n }\n}\n","import { existsSync } from 'fs'\nimport sherpa, { type OfflineTts, type Wave } from 'sherpa-onnx-node'\nimport { resolveModelFiles } from './modelFiles'\n\nexport interface SynthesizerOptions {\n modelDir: string\n numThreads?: number\n provider?: string\n debug?: boolean\n}\n\nconst DEFAULT_NUM_THREADS = 2\nconst DEFAULT_PROVIDER = 'cpu'\n// Voices already turned into an embedding, kept by the addon between calls\nconst VOICE_CACHE_CAPACITY = 10\n\n/**\n * One instance per configuration, shared by every call.\n *\n * The weights weigh close to two hundred megabytes, so holding a second copy\n * for a second call would cost more memory than the language model it sits\n * next to. Inference is bound to a couple of threads anyway, and two calls\n * generating at once compete for the same cores whether they share the model\n * or not.\n */\nconst synthesizers = new Map<string, Promise<OfflineTts>>()\n\nfunction cacheKey(options: SynthesizerOptions): string {\n return JSON.stringify([\n options.modelDir,\n options.numThreads ?? null,\n options.provider ?? null,\n options.debug ?? null,\n ])\n}\n\nexport function loadSynthesizer(\n options: SynthesizerOptions\n): Promise<OfflineTts> {\n const key = cacheKey(options)\n const existing = synthesizers.get(key)\n if (existing) return existing\n\n // createAsync loads the graphs on a worker thread, so setting a call up does\n // not hold the event loop for the half second it takes\n const loading = Promise.resolve().then(() =>\n sherpa.OfflineTts.createAsync({\n model: {\n pocket: {\n ...resolveModelFiles(options.modelDir),\n voiceEmbeddingCacheCapacity: VOICE_CACHE_CAPACITY,\n },\n numThreads: options.numThreads ?? DEFAULT_NUM_THREADS,\n provider: options.provider ?? DEFAULT_PROVIDER,\n debug: options.debug ?? false,\n },\n // Sentences are already split upstream, one per synthesis\n maxNumSentences: 1,\n })\n )\n\n // A failed load must not poison the cache, the next call retries it\n loading.catch(() => synthesizers.delete(key))\n\n synthesizers.set(key, loading)\n return loading\n}\n\n/**\n * Reference voices read from disk, kept by path.\n *\n * The samples are handed to the addon on every sentence, which is where the\n * voice embedding gets cached, so what is saved here is only the file read.\n */\nconst voices = new Map<string, Wave>()\n\nexport function loadVoice(path: string): Wave {\n const existing = voices.get(path)\n if (existing) return existing\n\n if (!existsSync(path)) {\n throw new Error(`Pocket TTS reference voice not found: \"${path}\"`)\n }\n\n const wave = sherpa.readWave(path)\n voices.set(path, wave)\n return wave\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,gBAA2B;AAC3B,kBAAqB;AAarB,IAAM,UAAU;AAAA,EACd,QAAQ;AAAA,EACR,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,SAAS;AAAA,EACT,iBAAiB;AACnB;AAEA,IAAM,aAAa;AAAA,EACjB,WAAW;AAAA,EACX,iBAAiB;AACnB;AAEO,SAAS,kBACd,UACwD;AACxD,QAAM,QAAgC,CAAC;AACvC,QAAM,UAAoB,CAAC;AAE3B,aAAW,CAAC,KAAK,IAAI,KAAK,OAAO,QAAQ,OAAO,GAAG;AACjD,UAAM,QAAQ,CAAC,GAAG,IAAI,cAAc,GAAG,IAAI,OAAO,EAC/C,IAAI,CAAC,aAAS,kBAAK,UAAU,IAAI,CAAC,EAClC,KAAK,oBAAU;AAClB,QAAI,MAAO,OAAM,GAAG,IAAI;AAAA,QACnB,SAAQ,KAAK,GAAG,IAAI,iBAAiB,IAAI,OAAO;AAAA,EACvD;AAEA,aAAW,CAAC,KAAK,IAAI,KAAK,OAAO,QAAQ,UAAU,GAAG;AACpD,UAAM,WAAO,kBAAK,UAAU,IAAI;AAChC,YAAI,sBAAW,IAAI,EAAG,OAAM,GAAG,IAAI;AAAA,QAC9B,SAAQ,KAAK,IAAI;AAAA,EACxB;AAEA,MAAI,QAAQ,SAAS,GAAG;AACtB,UAAM,IAAI;AAAA,MACR,wCAAwC,QAAQ,MAAM,QAAQ,KAAK,IAAI,CAAC;AAAA,IAE1E;AAAA,EACF;AAEA,SAAO;AACT;AAUO,IAAM,iBAAiB;AAAA,EAC5B,MAAM;AAAA,EACN,OAAO;AAAA,EACP,cAAc;AAChB;AAKO,SAAS,iBACd,UACA,OACQ;AACR,QAAM,UAAU,eAAe,KAA2B;AAC1D,SAAO,cAAU,kBAAK,UAAU,OAAO,IAAI;AAC7C;;;AChFA,oBAA4D;AAC5D,IAAAA,2BAA8C;;;ACD9C,IAAAC,aAA2B;AAC3B,8BAAmD;AAUnD,IAAM,sBAAsB;AAC5B,IAAM,mBAAmB;AAEzB,IAAM,uBAAuB;AAW7B,IAAM,eAAe,oBAAI,IAAiC;AAE1D,SAAS,SAAS,SAAqC;AACrD,SAAO,KAAK,UAAU;AAAA,IACpB,QAAQ;AAAA,IACR,QAAQ,cAAc;AAAA,IACtB,QAAQ,YAAY;AAAA,IACpB,QAAQ,SAAS;AAAA,EACnB,CAAC;AACH;AAEO,SAAS,gBACd,SACqB;AACrB,QAAM,MAAM,SAAS,OAAO;AAC5B,QAAM,WAAW,aAAa,IAAI,GAAG;AACrC,MAAI,SAAU,QAAO;AAIrB,QAAM,UAAU,QAAQ,QAAQ,EAAE;AAAA,IAAK,MACrC,wBAAAC,QAAO,WAAW,YAAY;AAAA,MAC5B,OAAO;AAAA,QACL,QAAQ;AAAA,UACN,GAAG,kBAAkB,QAAQ,QAAQ;AAAA,UACrC,6BAA6B;AAAA,QAC/B;AAAA,QACA,YAAY,QAAQ,cAAc;AAAA,QAClC,UAAU,QAAQ,YAAY;AAAA,QAC9B,OAAO,QAAQ,SAAS;AAAA,MAC1B;AAAA;AAAA,MAEA,iBAAiB;AAAA,IACnB,CAAC;AAAA,EACH;AAGA,UAAQ,MAAM,MAAM,aAAa,OAAO,GAAG,CAAC;AAE5C,eAAa,IAAI,KAAK,OAAO;AAC7B,SAAO;AACT;AAQA,IAAM,SAAS,oBAAI,IAAkB;AAE9B,SAAS,UAAU,MAAoB;AAC5C,QAAM,WAAW,OAAO,IAAI,IAAI;AAChC,MAAI,SAAU,QAAO;AAErB,MAAI,KAAC,uBAAW,IAAI,GAAG;AACrB,UAAM,IAAI,MAAM,0CAA0C,IAAI,GAAG;AAAA,EACnE;AAEA,QAAM,OAAO,wBAAAA,QAAO,SAAS,IAAI;AACjC,SAAO,IAAI,MAAM,IAAI;AACrB,SAAO;AACT;;;AD7BA,IAAM,gBAAoC;AAC1C,IAAM,oBAAoB;AAC1B,IAAM,2BAA2B;AACjC,IAAM,qBAAqB;AAEpB,IAAM,YAAN,cAAwB,0BAAY;AAAA,EACzC,YAA6B,SAA2B;AACtD,UAAM;AADqB;AAI3B,sBAAkB,QAAQ,QAAQ;AAElC,oBAAgB,KAAK,sBAAsB,CAAC,EAAE,MAAM,CAAC,UAAU;AAC7D,cAAQ,MAAM,qCAAqC,KAAK;AAAA,IAC1D,CAAC;AAED,QAAI,QAAQ,WAAW,OAAO;AAC5B,WAAK,OAAO,EACT,KAAK,MAAM,KAAK,IAAI,iBAAiB,CAAC,EACtC,MAAM,MAAM;AAAA,MAAC,CAAC;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,MAAgB,WACd,MACA,QACoB;AACpB,UAAM,cAAc,MAAM,gBAAgB,KAAK,sBAAsB,CAAC;AAEtE,QAAI,OAAO,QAAS;AAIpB,UAAM,YAAY,IAAI;AAAA,MACpB,YAAY;AAAA,MACZ;AAAA,IACF;AAEA,UAAM,YAAY,cAAc;AAAA,MAC9B;AAAA;AAAA;AAAA,MAGA,sBAAsB;AAAA,MACtB,kBAAkB,KAAK,oBAAoB;AAAA,MAC3C,YAAY,CAAC,EAAE,QAAQ,MAAM;AAC3B,cAAM,QAAQ,UAAU,YAAQ,8BAAe,OAAO,CAAC;AAEvD,eAAO,KAAK,UAAU,KAAK,IAAI,IAAI;AAAA,MACrC;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAc,SAAwB;AACpC,UAAM,cAAc,MAAM,gBAAgB,KAAK,sBAAsB,CAAC;AACtE,UAAM,YAAY,cAAc;AAAA,MAC9B,MAAM;AAAA,MACN,kBAAkB,KAAK,oBAAoB;AAAA,IAC7C,CAAC;AAAA,EACH;AAAA,EAEQ,sBAAwC;AAC9C,UAAM,EAAE,QAAQ,IAAI;AACpB,UAAM,QAAQ;AAAA,MACZ,iBAAiB,QAAQ,UAAU,QAAQ,SAAS,aAAa;AAAA,IACnE;AAEA,WAAO,IAAI,yBAAAC,QAAO,iBAAiB;AAAA,MACjC,OAAO,QAAQ;AAAA,MACf,UAAU,QAAQ,YAAY;AAAA,MAC9B,gBAAgB,MAAM;AAAA,MACtB,qBAAqB,MAAM;AAAA,MAC3B,OAAO;AAAA,QACL,yBACE,QAAQ,mBAAmB;AAAA,QAC7B,GAAI,QAAQ,SAAS,SAAY,CAAC,IAAI,EAAE,MAAM,QAAQ,KAAK;AAAA,MAC7D;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,wBAA4C;AAClD,WAAO;AAAA,MACL,UAAU,KAAK,QAAQ;AAAA,MACvB,YAAY,KAAK,QAAQ;AAAA,MACzB,UAAU,KAAK,QAAQ;AAAA,IACzB;AAAA,EACF;AACF;","names":["import_sherpa_onnx_node","import_fs","sherpa","sherpa"]}
package/dist/index.mjs ADDED
@@ -0,0 +1,179 @@
1
+ // src/modelFiles.ts
2
+ import { existsSync } from "fs";
3
+ import { join } from "path";
4
+ var WEIGHTS = {
5
+ lmFlow: "lm_flow",
6
+ lmMain: "lm_main",
7
+ encoder: "encoder",
8
+ decoder: "decoder",
9
+ textConditioner: "text_conditioner"
10
+ };
11
+ var JSON_FILES = {
12
+ vocabJson: "vocab.json",
13
+ tokenScoresJson: "token_scores.json"
14
+ };
15
+ function resolveModelFiles(modelDir) {
16
+ const files = {};
17
+ const missing = [];
18
+ for (const [key, base] of Object.entries(WEIGHTS)) {
19
+ const found = [`${base}.int8.onnx`, `${base}.onnx`].map((name) => join(modelDir, name)).find(existsSync);
20
+ if (found) files[key] = found;
21
+ else missing.push(`${base}.int8.onnx or ${base}.onnx`);
22
+ }
23
+ for (const [key, name] of Object.entries(JSON_FILES)) {
24
+ const path = join(modelDir, name);
25
+ if (existsSync(path)) files[key] = path;
26
+ else missing.push(name);
27
+ }
28
+ if (missing.length > 0) {
29
+ throw new Error(
30
+ `Pocket TTS model files not found in "${modelDir}": ${missing.join(", ")}. Download the archive from https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models`
31
+ );
32
+ }
33
+ return files;
34
+ }
35
+ var BUNDLED_VOICES = {
36
+ bria: "test_wavs/bria.wav",
37
+ loona: "test_wavs/loona.wav",
38
+ frenchAccent: "test_wavs/sample_fr_hibiki_crepes.wav"
39
+ };
40
+ function resolveVoicePath(modelDir, voice) {
41
+ const bundled = BUNDLED_VOICES[voice];
42
+ return bundled ? join(modelDir, bundled) : voice;
43
+ }
44
+
45
+ // src/PocketTTS.ts
46
+ import { float32ToPcm16, Pcm16Resampler, SentenceTTS } from "@micdrop/server";
47
+ import sherpa2 from "sherpa-onnx-node";
48
+
49
+ // src/synthesizer.ts
50
+ import { existsSync as existsSync2 } from "fs";
51
+ import sherpa from "sherpa-onnx-node";
52
+ var DEFAULT_NUM_THREADS = 2;
53
+ var DEFAULT_PROVIDER = "cpu";
54
+ var VOICE_CACHE_CAPACITY = 10;
55
+ var synthesizers = /* @__PURE__ */ new Map();
56
+ function cacheKey(options) {
57
+ return JSON.stringify([
58
+ options.modelDir,
59
+ options.numThreads ?? null,
60
+ options.provider ?? null,
61
+ options.debug ?? null
62
+ ]);
63
+ }
64
+ function loadSynthesizer(options) {
65
+ const key = cacheKey(options);
66
+ const existing = synthesizers.get(key);
67
+ if (existing) return existing;
68
+ const loading = Promise.resolve().then(
69
+ () => sherpa.OfflineTts.createAsync({
70
+ model: {
71
+ pocket: {
72
+ ...resolveModelFiles(options.modelDir),
73
+ voiceEmbeddingCacheCapacity: VOICE_CACHE_CAPACITY
74
+ },
75
+ numThreads: options.numThreads ?? DEFAULT_NUM_THREADS,
76
+ provider: options.provider ?? DEFAULT_PROVIDER,
77
+ debug: options.debug ?? false
78
+ },
79
+ // Sentences are already split upstream, one per synthesis
80
+ maxNumSentences: 1
81
+ })
82
+ );
83
+ loading.catch(() => synthesizers.delete(key));
84
+ synthesizers.set(key, loading);
85
+ return loading;
86
+ }
87
+ var voices = /* @__PURE__ */ new Map();
88
+ function loadVoice(path) {
89
+ const existing = voices.get(path);
90
+ if (existing) return existing;
91
+ if (!existsSync2(path)) {
92
+ throw new Error(`Pocket TTS reference voice not found: "${path}"`);
93
+ }
94
+ const wave = sherpa.readWave(path);
95
+ voices.set(path, wave);
96
+ return wave;
97
+ }
98
+
99
+ // src/PocketTTS.ts
100
+ var DEFAULT_VOICE = "bria";
101
+ var DEFAULT_NUM_STEPS = 2;
102
+ var DEFAULT_REFERENCE_LENGTH = 12;
103
+ var OUTPUT_SAMPLE_RATE = 16e3;
104
+ var PocketTTS = class extends SentenceTTS {
105
+ constructor(options) {
106
+ super();
107
+ this.options = options;
108
+ resolveModelFiles(options.modelDir);
109
+ loadSynthesizer(this.getSynthesizerOptions()).catch((error) => {
110
+ console.error("[PocketTTS] Failed to load model:", error);
111
+ });
112
+ if (options.warmup !== false) {
113
+ this.warmup().then(() => this.log("Model warmed up")).catch(() => {
114
+ });
115
+ }
116
+ }
117
+ async synthesize(text, signal) {
118
+ const synthesizer = await loadSynthesizer(this.getSynthesizerOptions());
119
+ if (signal.aborted) return;
120
+ const resampler = new Pcm16Resampler(
121
+ synthesizer.sampleRate,
122
+ OUTPUT_SAMPLE_RATE
123
+ );
124
+ await synthesizer.generateAsync({
125
+ text,
126
+ // The samples of a chunk live in native memory for the length of the
127
+ // callback, which is where they are copied into a PCM16 buffer
128
+ enableExternalBuffer: true,
129
+ generationConfig: this.getGenerationConfig(),
130
+ onProgress: ({ samples }) => {
131
+ const audio = resampler.process(float32ToPcm16(samples));
132
+ return this.emitAudio(audio) ? 1 : 0;
133
+ }
134
+ });
135
+ }
136
+ /**
137
+ * Speaks one short word to pay the first inference up front.
138
+ *
139
+ * It also builds the embedding of the voice, which the addon then keeps, so
140
+ * the first sentence of the call only pays for its own generation.
141
+ */
142
+ async warmup() {
143
+ const synthesizer = await loadSynthesizer(this.getSynthesizerOptions());
144
+ await synthesizer.generateAsync({
145
+ text: "Hello.",
146
+ generationConfig: this.getGenerationConfig()
147
+ });
148
+ }
149
+ getGenerationConfig() {
150
+ const { options } = this;
151
+ const voice = loadVoice(
152
+ resolveVoicePath(options.modelDir, options.voice ?? DEFAULT_VOICE)
153
+ );
154
+ return new sherpa2.GenerationConfig({
155
+ speed: options.speed,
156
+ numSteps: options.numSteps ?? DEFAULT_NUM_STEPS,
157
+ referenceAudio: voice.samples,
158
+ referenceSampleRate: voice.sampleRate,
159
+ extra: {
160
+ max_reference_audio_len: options.referenceLength ?? DEFAULT_REFERENCE_LENGTH,
161
+ ...options.seed === void 0 ? {} : { seed: options.seed }
162
+ }
163
+ });
164
+ }
165
+ getSynthesizerOptions() {
166
+ return {
167
+ modelDir: this.options.modelDir,
168
+ numThreads: this.options.numThreads,
169
+ provider: this.options.provider
170
+ };
171
+ }
172
+ };
173
+ export {
174
+ BUNDLED_VOICES,
175
+ PocketTTS,
176
+ resolveModelFiles,
177
+ resolveVoicePath
178
+ };
179
+ //# sourceMappingURL=index.mjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/modelFiles.ts","../src/PocketTTS.ts","../src/synthesizer.ts"],"sourcesContent":["import { existsSync } from 'fs'\nimport { join } from 'path'\nimport type { PocketModelConfig } from 'sherpa-onnx-node'\n\n/**\n * The files a Pocket TTS archive holds, resolved inside its folder.\n *\n * The model ships in two flavours, quantized and full precision, which name\n * their weights `lm_main.int8.onnx` and `lm_main.onnx`. Each part is looked up\n * in that order, so both archives work and a folder mixing them takes the\n * lighter file whenever there is one, which is what the quantized archive\n * itself does for its encoder and its text conditioner.\n */\n\nconst WEIGHTS = {\n lmFlow: 'lm_flow',\n lmMain: 'lm_main',\n encoder: 'encoder',\n decoder: 'decoder',\n textConditioner: 'text_conditioner',\n} as const\n\nconst JSON_FILES = {\n vocabJson: 'vocab.json',\n tokenScoresJson: 'token_scores.json',\n} as const\n\nexport function resolveModelFiles(\n modelDir: string\n): Omit<PocketModelConfig, 'voiceEmbeddingCacheCapacity'> {\n const files: Record<string, string> = {}\n const missing: string[] = []\n\n for (const [key, base] of Object.entries(WEIGHTS)) {\n const found = [`${base}.int8.onnx`, `${base}.onnx`]\n .map((name) => join(modelDir, name))\n .find(existsSync)\n if (found) files[key] = found\n else missing.push(`${base}.int8.onnx or ${base}.onnx`)\n }\n\n for (const [key, name] of Object.entries(JSON_FILES)) {\n const path = join(modelDir, name)\n if (existsSync(path)) files[key] = path\n else missing.push(name)\n }\n\n if (missing.length > 0) {\n throw new Error(\n `Pocket TTS model files not found in \"${modelDir}\": ${missing.join(', ')}. ` +\n 'Download the archive from https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models'\n )\n }\n\n return files as Omit<PocketModelConfig, 'voiceEmbeddingCacheCapacity'>\n}\n\n/**\n * The reference voices the archive ships in its `test_wavs` folder.\n *\n * `bria` is a woman reading calmly for forty seconds, `loona` is a one second\n * clip, which is the shortest a reference can reasonably be, and\n * `frenchAccent` is a French speaker, cloned into English words: the language\n * comes from the text, the timbre and the accent from the recording.\n */\nexport const BUNDLED_VOICES = {\n bria: 'test_wavs/bria.wav',\n loona: 'test_wavs/loona.wav',\n frenchAccent: 'test_wavs/sample_fr_hibiki_crepes.wav',\n} as const\n\nexport type PocketBundledVoice = keyof typeof BUNDLED_VOICES\n\n/** Turns a bundled voice name into a path, and leaves a path as it is. */\nexport function resolveVoicePath(\n modelDir: string,\n voice: PocketBundledVoice | (string & {})\n): string {\n const bundled = BUNDLED_VOICES[voice as PocketBundledVoice]\n return bundled ? join(modelDir, bundled) : voice\n}\n","import { float32ToPcm16, Pcm16Resampler, SentenceTTS } from '@micdrop/server'\nimport sherpa, { type GenerationConfig } from 'sherpa-onnx-node'\nimport {\n PocketBundledVoice,\n resolveModelFiles,\n resolveVoicePath,\n} from './modelFiles'\nimport { loadSynthesizer, loadVoice, SynthesizerOptions } from './synthesizer'\n\n/**\n * Local Pocket TTS text to speech, running in the Node process through the\n * sherpa-onnx addon and ONNX Runtime. Nothing leaves the machine and no server\n * has to be started next to it.\n *\n * Pocket TTS is a 100 million parameter model from Kyutai, built for the CPU,\n * that clones the voice of a few seconds of reference audio. It generates\n * several times faster than real time and hands its audio over while it is\n * still writing the rest, so the first words of a sentence reach the client in\n * about three hundred milliseconds.\n *\n * The weights are downloaded separately, see the README.\n *\n * @see https://github.com/kyutai-labs/pocket-tts\n * @see https://k2-fsa.github.io/sherpa/onnx/tts/pocket.html\n */\n\nexport interface PocketTTSOptions {\n /** Folder holding the model files, as extracted from the archive. */\n modelDir: string\n\n /**\n * Voice to clone: a bundled name (\"bria\"), or the path to a wav file\n * carrying a few seconds of the voice to speak with.\n */\n voice?: PocketBundledVoice | (string & {})\n\n /** Speech rate, 1 being the natural pace of the voice. */\n speed?: number\n\n /** Sampling steps per frame, higher sounding better and generating slower. */\n numSteps?: number\n\n /** Seconds of the reference audio the voice is built from. */\n referenceLength?: number\n\n /** Sampling seed, so the same sentence comes out the same way. */\n seed?: number\n\n /** Threads given to ONNX Runtime, 2 by default. */\n numThreads?: number\n\n /** Execution provider passed to ONNX Runtime, \"cpu\" by default. */\n provider?: 'cpu' | 'cuda' | 'coreml'\n\n /** Synthesizes one word at startup, on by default. */\n warmup?: boolean\n}\n\nconst DEFAULT_VOICE: PocketBundledVoice = 'bria'\nconst DEFAULT_NUM_STEPS = 2\nconst DEFAULT_REFERENCE_LENGTH = 12\nconst OUTPUT_SAMPLE_RATE = 16000 // Rate expected by the Micdrop client\n\nexport class PocketTTS extends SentenceTTS {\n constructor(private readonly options: PocketTTSOptions) {\n super()\n\n // Fail at startup rather than on the first sentence of the call\n resolveModelFiles(options.modelDir)\n\n loadSynthesizer(this.getSynthesizerOptions()).catch((error) => {\n console.error('[PocketTTS] Failed to load model:', error)\n })\n\n if (options.warmup !== false) {\n this.warmup()\n .then(() => this.log('Model warmed up'))\n .catch(() => {})\n }\n }\n\n protected async synthesize(\n text: string,\n signal: AbortSignal\n ): Promise<undefined> {\n const synthesizer = await loadSynthesizer(this.getSynthesizerOptions())\n // The utterance may have been cancelled while the model was loading\n if (signal.aborted) return\n\n // One resampler for the whole sentence: its chunks are a single continuous\n // stream, and the resampler carries its fractional position across them\n const resampler = new Pcm16Resampler(\n synthesizer.sampleRate,\n OUTPUT_SAMPLE_RATE\n )\n\n await synthesizer.generateAsync({\n text,\n // The samples of a chunk live in native memory for the length of the\n // callback, which is where they are copied into a PCM16 buffer\n enableExternalBuffer: true,\n generationConfig: this.getGenerationConfig(),\n onProgress: ({ samples }) => {\n const audio = resampler.process(float32ToPcm16(samples))\n // Stop generating what nobody is going to hear\n return this.emitAudio(audio) ? 1 : 0\n },\n })\n }\n\n /**\n * Speaks one short word to pay the first inference up front.\n *\n * It also builds the embedding of the voice, which the addon then keeps, so\n * the first sentence of the call only pays for its own generation.\n */\n private async warmup(): Promise<void> {\n const synthesizer = await loadSynthesizer(this.getSynthesizerOptions())\n await synthesizer.generateAsync({\n text: 'Hello.',\n generationConfig: this.getGenerationConfig(),\n })\n }\n\n private getGenerationConfig(): GenerationConfig {\n const { options } = this\n const voice = loadVoice(\n resolveVoicePath(options.modelDir, options.voice ?? DEFAULT_VOICE)\n )\n\n return new sherpa.GenerationConfig({\n speed: options.speed,\n numSteps: options.numSteps ?? DEFAULT_NUM_STEPS,\n referenceAudio: voice.samples,\n referenceSampleRate: voice.sampleRate,\n extra: {\n max_reference_audio_len:\n options.referenceLength ?? DEFAULT_REFERENCE_LENGTH,\n ...(options.seed === undefined ? {} : { seed: options.seed }),\n },\n })\n }\n\n private getSynthesizerOptions(): SynthesizerOptions {\n return {\n modelDir: this.options.modelDir,\n numThreads: this.options.numThreads,\n provider: this.options.provider,\n }\n }\n}\n","import { existsSync } from 'fs'\nimport sherpa, { type OfflineTts, type Wave } from 'sherpa-onnx-node'\nimport { resolveModelFiles } from './modelFiles'\n\nexport interface SynthesizerOptions {\n modelDir: string\n numThreads?: number\n provider?: string\n debug?: boolean\n}\n\nconst DEFAULT_NUM_THREADS = 2\nconst DEFAULT_PROVIDER = 'cpu'\n// Voices already turned into an embedding, kept by the addon between calls\nconst VOICE_CACHE_CAPACITY = 10\n\n/**\n * One instance per configuration, shared by every call.\n *\n * The weights weigh close to two hundred megabytes, so holding a second copy\n * for a second call would cost more memory than the language model it sits\n * next to. Inference is bound to a couple of threads anyway, and two calls\n * generating at once compete for the same cores whether they share the model\n * or not.\n */\nconst synthesizers = new Map<string, Promise<OfflineTts>>()\n\nfunction cacheKey(options: SynthesizerOptions): string {\n return JSON.stringify([\n options.modelDir,\n options.numThreads ?? null,\n options.provider ?? null,\n options.debug ?? null,\n ])\n}\n\nexport function loadSynthesizer(\n options: SynthesizerOptions\n): Promise<OfflineTts> {\n const key = cacheKey(options)\n const existing = synthesizers.get(key)\n if (existing) return existing\n\n // createAsync loads the graphs on a worker thread, so setting a call up does\n // not hold the event loop for the half second it takes\n const loading = Promise.resolve().then(() =>\n sherpa.OfflineTts.createAsync({\n model: {\n pocket: {\n ...resolveModelFiles(options.modelDir),\n voiceEmbeddingCacheCapacity: VOICE_CACHE_CAPACITY,\n },\n numThreads: options.numThreads ?? DEFAULT_NUM_THREADS,\n provider: options.provider ?? DEFAULT_PROVIDER,\n debug: options.debug ?? false,\n },\n // Sentences are already split upstream, one per synthesis\n maxNumSentences: 1,\n })\n )\n\n // A failed load must not poison the cache, the next call retries it\n loading.catch(() => synthesizers.delete(key))\n\n synthesizers.set(key, loading)\n return loading\n}\n\n/**\n * Reference voices read from disk, kept by path.\n *\n * The samples are handed to the addon on every sentence, which is where the\n * voice embedding gets cached, so what is saved here is only the file read.\n */\nconst voices = new Map<string, Wave>()\n\nexport function loadVoice(path: string): Wave {\n const existing = voices.get(path)\n if (existing) return existing\n\n if (!existsSync(path)) {\n throw new Error(`Pocket TTS reference voice not found: \"${path}\"`)\n }\n\n const wave = sherpa.readWave(path)\n voices.set(path, wave)\n return wave\n}\n"],"mappings":";AAAA,SAAS,kBAAkB;AAC3B,SAAS,YAAY;AAarB,IAAM,UAAU;AAAA,EACd,QAAQ;AAAA,EACR,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,SAAS;AAAA,EACT,iBAAiB;AACnB;AAEA,IAAM,aAAa;AAAA,EACjB,WAAW;AAAA,EACX,iBAAiB;AACnB;AAEO,SAAS,kBACd,UACwD;AACxD,QAAM,QAAgC,CAAC;AACvC,QAAM,UAAoB,CAAC;AAE3B,aAAW,CAAC,KAAK,IAAI,KAAK,OAAO,QAAQ,OAAO,GAAG;AACjD,UAAM,QAAQ,CAAC,GAAG,IAAI,cAAc,GAAG,IAAI,OAAO,EAC/C,IAAI,CAAC,SAAS,KAAK,UAAU,IAAI,CAAC,EAClC,KAAK,UAAU;AAClB,QAAI,MAAO,OAAM,GAAG,IAAI;AAAA,QACnB,SAAQ,KAAK,GAAG,IAAI,iBAAiB,IAAI,OAAO;AAAA,EACvD;AAEA,aAAW,CAAC,KAAK,IAAI,KAAK,OAAO,QAAQ,UAAU,GAAG;AACpD,UAAM,OAAO,KAAK,UAAU,IAAI;AAChC,QAAI,WAAW,IAAI,EAAG,OAAM,GAAG,IAAI;AAAA,QAC9B,SAAQ,KAAK,IAAI;AAAA,EACxB;AAEA,MAAI,QAAQ,SAAS,GAAG;AACtB,UAAM,IAAI;AAAA,MACR,wCAAwC,QAAQ,MAAM,QAAQ,KAAK,IAAI,CAAC;AAAA,IAE1E;AAAA,EACF;AAEA,SAAO;AACT;AAUO,IAAM,iBAAiB;AAAA,EAC5B,MAAM;AAAA,EACN,OAAO;AAAA,EACP,cAAc;AAChB;AAKO,SAAS,iBACd,UACA,OACQ;AACR,QAAM,UAAU,eAAe,KAA2B;AAC1D,SAAO,UAAU,KAAK,UAAU,OAAO,IAAI;AAC7C;;;AChFA,SAAS,gBAAgB,gBAAgB,mBAAmB;AAC5D,OAAOA,aAAuC;;;ACD9C,SAAS,cAAAC,mBAAkB;AAC3B,OAAO,YAA4C;AAUnD,IAAM,sBAAsB;AAC5B,IAAM,mBAAmB;AAEzB,IAAM,uBAAuB;AAW7B,IAAM,eAAe,oBAAI,IAAiC;AAE1D,SAAS,SAAS,SAAqC;AACrD,SAAO,KAAK,UAAU;AAAA,IACpB,QAAQ;AAAA,IACR,QAAQ,cAAc;AAAA,IACtB,QAAQ,YAAY;AAAA,IACpB,QAAQ,SAAS;AAAA,EACnB,CAAC;AACH;AAEO,SAAS,gBACd,SACqB;AACrB,QAAM,MAAM,SAAS,OAAO;AAC5B,QAAM,WAAW,aAAa,IAAI,GAAG;AACrC,MAAI,SAAU,QAAO;AAIrB,QAAM,UAAU,QAAQ,QAAQ,EAAE;AAAA,IAAK,MACrC,OAAO,WAAW,YAAY;AAAA,MAC5B,OAAO;AAAA,QACL,QAAQ;AAAA,UACN,GAAG,kBAAkB,QAAQ,QAAQ;AAAA,UACrC,6BAA6B;AAAA,QAC/B;AAAA,QACA,YAAY,QAAQ,cAAc;AAAA,QAClC,UAAU,QAAQ,YAAY;AAAA,QAC9B,OAAO,QAAQ,SAAS;AAAA,MAC1B;AAAA;AAAA,MAEA,iBAAiB;AAAA,IACnB,CAAC;AAAA,EACH;AAGA,UAAQ,MAAM,MAAM,aAAa,OAAO,GAAG,CAAC;AAE5C,eAAa,IAAI,KAAK,OAAO;AAC7B,SAAO;AACT;AAQA,IAAM,SAAS,oBAAI,IAAkB;AAE9B,SAAS,UAAU,MAAoB;AAC5C,QAAM,WAAW,OAAO,IAAI,IAAI;AAChC,MAAI,SAAU,QAAO;AAErB,MAAI,CAACC,YAAW,IAAI,GAAG;AACrB,UAAM,IAAI,MAAM,0CAA0C,IAAI,GAAG;AAAA,EACnE;AAEA,QAAM,OAAO,OAAO,SAAS,IAAI;AACjC,SAAO,IAAI,MAAM,IAAI;AACrB,SAAO;AACT;;;AD7BA,IAAM,gBAAoC;AAC1C,IAAM,oBAAoB;AAC1B,IAAM,2BAA2B;AACjC,IAAM,qBAAqB;AAEpB,IAAM,YAAN,cAAwB,YAAY;AAAA,EACzC,YAA6B,SAA2B;AACtD,UAAM;AADqB;AAI3B,sBAAkB,QAAQ,QAAQ;AAElC,oBAAgB,KAAK,sBAAsB,CAAC,EAAE,MAAM,CAAC,UAAU;AAC7D,cAAQ,MAAM,qCAAqC,KAAK;AAAA,IAC1D,CAAC;AAED,QAAI,QAAQ,WAAW,OAAO;AAC5B,WAAK,OAAO,EACT,KAAK,MAAM,KAAK,IAAI,iBAAiB,CAAC,EACtC,MAAM,MAAM;AAAA,MAAC,CAAC;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,MAAgB,WACd,MACA,QACoB;AACpB,UAAM,cAAc,MAAM,gBAAgB,KAAK,sBAAsB,CAAC;AAEtE,QAAI,OAAO,QAAS;AAIpB,UAAM,YAAY,IAAI;AAAA,MACpB,YAAY;AAAA,MACZ;AAAA,IACF;AAEA,UAAM,YAAY,cAAc;AAAA,MAC9B;AAAA;AAAA;AAAA,MAGA,sBAAsB;AAAA,MACtB,kBAAkB,KAAK,oBAAoB;AAAA,MAC3C,YAAY,CAAC,EAAE,QAAQ,MAAM;AAC3B,cAAM,QAAQ,UAAU,QAAQ,eAAe,OAAO,CAAC;AAEvD,eAAO,KAAK,UAAU,KAAK,IAAI,IAAI;AAAA,MACrC;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAc,SAAwB;AACpC,UAAM,cAAc,MAAM,gBAAgB,KAAK,sBAAsB,CAAC;AACtE,UAAM,YAAY,cAAc;AAAA,MAC9B,MAAM;AAAA,MACN,kBAAkB,KAAK,oBAAoB;AAAA,IAC7C,CAAC;AAAA,EACH;AAAA,EAEQ,sBAAwC;AAC9C,UAAM,EAAE,QAAQ,IAAI;AACpB,UAAM,QAAQ;AAAA,MACZ,iBAAiB,QAAQ,UAAU,QAAQ,SAAS,aAAa;AAAA,IACnE;AAEA,WAAO,IAAIC,QAAO,iBAAiB;AAAA,MACjC,OAAO,QAAQ;AAAA,MACf,UAAU,QAAQ,YAAY;AAAA,MAC9B,gBAAgB,MAAM;AAAA,MACtB,qBAAqB,MAAM;AAAA,MAC3B,OAAO;AAAA,QACL,yBACE,QAAQ,mBAAmB;AAAA,QAC7B,GAAI,QAAQ,SAAS,SAAY,CAAC,IAAI,EAAE,MAAM,QAAQ,KAAK;AAAA,MAC7D;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,wBAA4C;AAClD,WAAO;AAAA,MACL,UAAU,KAAK,QAAQ;AAAA,MACvB,YAAY,KAAK,QAAQ;AAAA,MACzB,UAAU,KAAK,QAAQ;AAAA,IACzB;AAAA,EACF;AACF;","names":["sherpa","existsSync","existsSync","sherpa"]}
package/package.json ADDED
@@ -0,0 +1,50 @@
1
+ {
2
+ "name": "@micdrop/pocket-tts",
3
+ "version": "1.0.0",
4
+ "description": "Local Pocket TTS text to speech for @micdrop/server",
5
+ "author": "Godefroy de Compreignac",
6
+ "license": "MIT",
7
+ "main": "./dist/index.js",
8
+ "module": "./dist/index.mjs",
9
+ "types": "./dist/index.d.ts",
10
+ "scripts": {
11
+ "build": "tsup",
12
+ "dev": "tsup --watch",
13
+ "clean": "rm -rf dist",
14
+ "typecheck": "tsc --noEmit",
15
+ "prepublishOnly": "npm run build"
16
+ },
17
+ "files": [
18
+ "dist"
19
+ ],
20
+ "repository": {
21
+ "type": "git",
22
+ "url": "git+https://github.com/Godefroy/micdrop.git",
23
+ "directory": "packages/pocket-tts"
24
+ },
25
+ "publishConfig": {
26
+ "access": "public"
27
+ },
28
+ "keywords": [
29
+ "microphone",
30
+ "audio",
31
+ "voice",
32
+ "conversation",
33
+ "pocket-tts",
34
+ "kyutai",
35
+ "local",
36
+ "offline",
37
+ "tts"
38
+ ],
39
+ "dependencies": {
40
+ "sherpa-onnx-node": "^1.13.6"
41
+ },
42
+ "peerDependencies": {
43
+ "@micdrop/server": ">=2.4.0"
44
+ },
45
+ "devDependencies": {
46
+ "@types/node": "^22.19.13",
47
+ "tsup": "^8.5.1",
48
+ "typescript": "^5.9.3"
49
+ }
50
+ }