@micdrop/pocket-tts 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -1
- package/dist/index.js +15 -20
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +11 -16
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -34,7 +34,7 @@ The archive weighs 98 MB and takes 190 MB on disk. A full precision archive,
|
|
|
34
34
|
`sherpa-onnx-pocket-tts-2026-01-26.tar.bz2`, exists next to it and is read the
|
|
35
35
|
same way.
|
|
36
36
|
|
|
37
|
-
## Usage
|
|
37
|
+
## Usage with MicdropServer
|
|
38
38
|
|
|
39
39
|
```typescript
|
|
40
40
|
import { PocketTTS } from '@micdrop/pocket-tts'
|
|
@@ -52,6 +52,33 @@ new MicdropServer(socket, {
|
|
|
52
52
|
})
|
|
53
53
|
```
|
|
54
54
|
|
|
55
|
+
## Usage without MicdropServer
|
|
56
|
+
|
|
57
|
+
```typescript
|
|
58
|
+
import { PocketTTS } from '@micdrop/pocket-tts'
|
|
59
|
+
import { Readable } from 'stream'
|
|
60
|
+
|
|
61
|
+
const tts = new PocketTTS({
|
|
62
|
+
modelDir: './sherpa-onnx-pocket-tts-int8-2026-01-26',
|
|
63
|
+
voice: 'bria',
|
|
64
|
+
})
|
|
65
|
+
|
|
66
|
+
// Audio is raw PCM, 16 bits, 16 kHz, mono
|
|
67
|
+
tts.on('Audio', (chunk) => console.log('Audio:', chunk.length, 'bytes'))
|
|
68
|
+
tts.on('Failed', (texts) => console.error('Failed:', texts))
|
|
69
|
+
|
|
70
|
+
tts.speak(Readable.from(['Hello! ', 'What can I do for you?']))
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## Events
|
|
74
|
+
|
|
75
|
+
| Event | Payload | Description |
|
|
76
|
+
| -------- | ---------- | ------------------------------------------------------------------------ |
|
|
77
|
+
| `Audio` | `Buffer` | A chunk of audio, PCM 16 bits, 16 kHz, mono, ready to be played. |
|
|
78
|
+
| `Failed` | `string[]` | Synthesis gave up after its retries, with the text that stayed unspoken. |
|
|
79
|
+
|
|
80
|
+
See the [TTS interface](https://micdrop.dev/docs/ai-integration/custom-integrations/custom-tts) for the full contract.
|
|
81
|
+
|
|
55
82
|
## Voices
|
|
56
83
|
|
|
57
84
|
The voice comes from a reference recording rather than from a catalog: a few
|
package/dist/index.js
CHANGED
|
@@ -82,29 +82,27 @@ function resolveVoicePath(modelDir, voice) {
|
|
|
82
82
|
}
|
|
83
83
|
|
|
84
84
|
// src/PocketTTS.ts
|
|
85
|
-
var
|
|
85
|
+
var import_server2 = require("@micdrop/server");
|
|
86
86
|
var import_sherpa_onnx_node2 = __toESM(require("sherpa-onnx-node"));
|
|
87
87
|
|
|
88
88
|
// src/synthesizer.ts
|
|
89
|
+
var import_server = require("@micdrop/server");
|
|
89
90
|
var import_fs2 = require("fs");
|
|
90
91
|
var import_sherpa_onnx_node = __toESM(require("sherpa-onnx-node"));
|
|
91
92
|
var DEFAULT_NUM_THREADS = 2;
|
|
92
93
|
var DEFAULT_PROVIDER = "cpu";
|
|
93
94
|
var VOICE_CACHE_CAPACITY = 10;
|
|
94
|
-
var synthesizers =
|
|
95
|
-
function cacheKey(options) {
|
|
96
|
-
return JSON.stringify([
|
|
97
|
-
options.modelDir,
|
|
98
|
-
options.numThreads ?? null,
|
|
99
|
-
options.provider ?? null,
|
|
100
|
-
options.debug ?? null
|
|
101
|
-
]);
|
|
102
|
-
}
|
|
95
|
+
var synthesizers = new import_server.SharedInstances();
|
|
103
96
|
function loadSynthesizer(options) {
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
97
|
+
return synthesizers.load(
|
|
98
|
+
[
|
|
99
|
+
options.modelDir,
|
|
100
|
+
options.numThreads ?? null,
|
|
101
|
+
options.provider ?? null,
|
|
102
|
+
options.debug ?? null
|
|
103
|
+
],
|
|
104
|
+
// createAsync loads the graphs on a worker thread, so setting a call up
|
|
105
|
+
// does not hold the event loop for the half second it takes
|
|
108
106
|
() => import_sherpa_onnx_node.default.OfflineTts.createAsync({
|
|
109
107
|
model: {
|
|
110
108
|
pocket: {
|
|
@@ -119,9 +117,6 @@ function loadSynthesizer(options) {
|
|
|
119
117
|
maxNumSentences: 1
|
|
120
118
|
})
|
|
121
119
|
);
|
|
122
|
-
loading.catch(() => synthesizers.delete(key));
|
|
123
|
-
synthesizers.set(key, loading);
|
|
124
|
-
return loading;
|
|
125
120
|
}
|
|
126
121
|
var voices = /* @__PURE__ */ new Map();
|
|
127
122
|
function loadVoice(path) {
|
|
@@ -140,7 +135,7 @@ var DEFAULT_VOICE = "bria";
|
|
|
140
135
|
var DEFAULT_NUM_STEPS = 2;
|
|
141
136
|
var DEFAULT_REFERENCE_LENGTH = 12;
|
|
142
137
|
var OUTPUT_SAMPLE_RATE = 16e3;
|
|
143
|
-
var PocketTTS = class extends
|
|
138
|
+
var PocketTTS = class extends import_server2.SentenceTTS {
|
|
144
139
|
constructor(options) {
|
|
145
140
|
super();
|
|
146
141
|
this.options = options;
|
|
@@ -156,7 +151,7 @@ var PocketTTS = class extends import_server.SentenceTTS {
|
|
|
156
151
|
async synthesize(text, signal) {
|
|
157
152
|
const synthesizer = await loadSynthesizer(this.getSynthesizerOptions());
|
|
158
153
|
if (signal.aborted) return;
|
|
159
|
-
const resampler = new
|
|
154
|
+
const resampler = new import_server2.Pcm16Resampler(
|
|
160
155
|
synthesizer.sampleRate,
|
|
161
156
|
OUTPUT_SAMPLE_RATE
|
|
162
157
|
);
|
|
@@ -167,7 +162,7 @@ var PocketTTS = class extends import_server.SentenceTTS {
|
|
|
167
162
|
enableExternalBuffer: true,
|
|
168
163
|
generationConfig: this.getGenerationConfig(),
|
|
169
164
|
onProgress: ({ samples }) => {
|
|
170
|
-
const audio = resampler.process((0,
|
|
165
|
+
const audio = resampler.process((0, import_server2.float32ToPcm16)(samples));
|
|
171
166
|
return this.emitAudio(audio) ? 1 : 0;
|
|
172
167
|
}
|
|
173
168
|
});
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/index.ts","../src/modelFiles.ts","../src/PocketTTS.ts","../src/synthesizer.ts"],"sourcesContent":["export * from './modelFiles'\nexport * from './PocketTTS'\n","import { existsSync } from 'fs'\nimport { join } from 'path'\nimport type { PocketModelConfig } from 'sherpa-onnx-node'\n\n/**\n * The files a Pocket TTS archive holds, resolved inside its folder.\n *\n * The model ships in two flavours, quantized and full precision, which name\n * their weights `lm_main.int8.onnx` and `lm_main.onnx`. Each part is looked up\n * in that order, so both archives work and a folder mixing them takes the\n * lighter file whenever there is one, which is what the quantized archive\n * itself does for its encoder and its text conditioner.\n */\n\nconst WEIGHTS = {\n lmFlow: 'lm_flow',\n lmMain: 'lm_main',\n encoder: 'encoder',\n decoder: 'decoder',\n textConditioner: 'text_conditioner',\n} as const\n\nconst JSON_FILES = {\n vocabJson: 'vocab.json',\n tokenScoresJson: 'token_scores.json',\n} as const\n\nexport function resolveModelFiles(\n modelDir: string\n): Omit<PocketModelConfig, 'voiceEmbeddingCacheCapacity'> {\n const files: Record<string, string> = {}\n const missing: string[] = []\n\n for (const [key, base] of Object.entries(WEIGHTS)) {\n const found = [`${base}.int8.onnx`, `${base}.onnx`]\n .map((name) => join(modelDir, name))\n .find(existsSync)\n if (found) files[key] = found\n else missing.push(`${base}.int8.onnx or ${base}.onnx`)\n }\n\n for (const [key, name] of Object.entries(JSON_FILES)) {\n const path = join(modelDir, name)\n if (existsSync(path)) files[key] = path\n else missing.push(name)\n }\n\n if (missing.length > 0) {\n throw new Error(\n `Pocket TTS model files not found in \"${modelDir}\": ${missing.join(', ')}. ` +\n 'Download the archive from https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models'\n )\n }\n\n return files as Omit<PocketModelConfig, 'voiceEmbeddingCacheCapacity'>\n}\n\n/**\n * The reference voices the archive ships in its `test_wavs` folder.\n *\n * `bria` is a woman reading calmly for forty seconds, `loona` is a one second\n * clip, which is the shortest a reference can reasonably be, and\n * `frenchAccent` is a French speaker, cloned into English words: the language\n * comes from the text, the timbre and the accent from the recording.\n */\nexport const BUNDLED_VOICES = {\n bria: 'test_wavs/bria.wav',\n loona: 'test_wavs/loona.wav',\n frenchAccent: 'test_wavs/sample_fr_hibiki_crepes.wav',\n} as const\n\nexport type PocketBundledVoice = keyof typeof BUNDLED_VOICES\n\n/** Turns a bundled voice name into a path, and leaves a path as it is. */\nexport function resolveVoicePath(\n modelDir: string,\n voice: PocketBundledVoice | (string & {})\n): string {\n const bundled = BUNDLED_VOICES[voice as PocketBundledVoice]\n return bundled ? join(modelDir, bundled) : voice\n}\n","import { float32ToPcm16, Pcm16Resampler, SentenceTTS } from '@micdrop/server'\nimport sherpa, { type GenerationConfig } from 'sherpa-onnx-node'\nimport {\n PocketBundledVoice,\n resolveModelFiles,\n resolveVoicePath,\n} from './modelFiles'\nimport { loadSynthesizer, loadVoice, SynthesizerOptions } from './synthesizer'\n\n/**\n * Local Pocket TTS text to speech, running in the Node process through the\n * sherpa-onnx addon and ONNX Runtime. Nothing leaves the machine and no server\n * has to be started next to it.\n *\n * Pocket TTS is a 100 million parameter model from Kyutai, built for the CPU,\n * that clones the voice of a few seconds of reference audio. It generates\n * several times faster than real time and hands its audio over while it is\n * still writing the rest, so the first words of a sentence reach the client in\n * about three hundred milliseconds.\n *\n * The weights are downloaded separately, see the README.\n *\n * @see https://github.com/kyutai-labs/pocket-tts\n * @see https://k2-fsa.github.io/sherpa/onnx/tts/pocket.html\n */\n\nexport interface PocketTTSOptions {\n /** Folder holding the model files, as extracted from the archive. */\n modelDir: string\n\n /**\n * Voice to clone: a bundled name (\"bria\"), or the path to a wav file\n * carrying a few seconds of the voice to speak with.\n */\n voice?: PocketBundledVoice | (string & {})\n\n /** Speech rate, 1 being the natural pace of the voice. */\n speed?: number\n\n /** Sampling steps per frame, higher sounding better and generating slower. */\n numSteps?: number\n\n /** Seconds of the reference audio the voice is built from. */\n referenceLength?: number\n\n /** Sampling seed, so the same sentence comes out the same way. */\n seed?: number\n\n /** Threads given to ONNX Runtime, 2 by default. */\n numThreads?: number\n\n /** Execution provider passed to ONNX Runtime, \"cpu\" by default. */\n provider?: 'cpu' | 'cuda' | 'coreml'\n\n /** Synthesizes one word at startup, on by default. */\n warmup?: boolean\n}\n\nconst DEFAULT_VOICE: PocketBundledVoice = 'bria'\nconst DEFAULT_NUM_STEPS = 2\nconst DEFAULT_REFERENCE_LENGTH = 12\nconst OUTPUT_SAMPLE_RATE = 16000 // Rate expected by the Micdrop client\n\nexport class PocketTTS extends SentenceTTS {\n constructor(private readonly options: PocketTTSOptions) {\n super()\n\n // Fail at startup rather than on the first sentence of the call\n resolveModelFiles(options.modelDir)\n\n loadSynthesizer(this.getSynthesizerOptions()).catch((error) => {\n console.error('[PocketTTS] Failed to load model:', error)\n })\n\n if (options.warmup !== false) {\n this.warmup()\n .then(() => this.log('Model warmed up'))\n .catch(() => {})\n }\n }\n\n protected async synthesize(\n text: string,\n signal: AbortSignal\n ): Promise<undefined> {\n const synthesizer = await loadSynthesizer(this.getSynthesizerOptions())\n // The utterance may have been cancelled while the model was loading\n if (signal.aborted) return\n\n // One resampler for the whole sentence: its chunks are a single continuous\n // stream, and the resampler carries its fractional position across them\n const resampler = new Pcm16Resampler(\n synthesizer.sampleRate,\n OUTPUT_SAMPLE_RATE\n )\n\n await synthesizer.generateAsync({\n text,\n // The samples of a chunk live in native memory for the length of the\n // callback, which is where they are copied into a PCM16 buffer\n enableExternalBuffer: true,\n generationConfig: this.getGenerationConfig(),\n onProgress: ({ samples }) => {\n const audio = resampler.process(float32ToPcm16(samples))\n // Stop generating what nobody is going to hear\n return this.emitAudio(audio) ? 1 : 0\n },\n })\n }\n\n /**\n * Speaks one short word to pay the first inference up front.\n *\n * It also builds the embedding of the voice, which the addon then keeps, so\n * the first sentence of the call only pays for its own generation.\n */\n private async warmup(): Promise<void> {\n const synthesizer = await loadSynthesizer(this.getSynthesizerOptions())\n await synthesizer.generateAsync({\n text: 'Hello.',\n generationConfig: this.getGenerationConfig(),\n })\n }\n\n private getGenerationConfig(): GenerationConfig {\n const { options } = this\n const voice = loadVoice(\n resolveVoicePath(options.modelDir, options.voice ?? DEFAULT_VOICE)\n )\n\n return new sherpa.GenerationConfig({\n speed: options.speed,\n numSteps: options.numSteps ?? DEFAULT_NUM_STEPS,\n referenceAudio: voice.samples,\n referenceSampleRate: voice.sampleRate,\n extra: {\n max_reference_audio_len:\n options.referenceLength ?? DEFAULT_REFERENCE_LENGTH,\n ...(options.seed === undefined ? {} : { seed: options.seed }),\n },\n })\n }\n\n private getSynthesizerOptions(): SynthesizerOptions {\n return {\n modelDir: this.options.modelDir,\n numThreads: this.options.numThreads,\n provider: this.options.provider,\n }\n }\n}\n","import { existsSync } from 'fs'\nimport sherpa, { type OfflineTts, type Wave } from 'sherpa-onnx-node'\nimport { resolveModelFiles } from './modelFiles'\n\nexport interface SynthesizerOptions {\n modelDir: string\n numThreads?: number\n provider?: string\n debug?: boolean\n}\n\nconst DEFAULT_NUM_THREADS = 2\nconst DEFAULT_PROVIDER = 'cpu'\n// Voices already turned into an embedding, kept by the addon between calls\nconst VOICE_CACHE_CAPACITY = 10\n\n/**\n * One instance per configuration, shared by every call.\n *\n * The weights weigh close to two hundred megabytes, so holding a second copy\n * for a second call would cost more memory than the language model it sits\n * next to. Inference is bound to a couple of threads anyway, and two calls\n * generating at once compete for the same cores whether they share the model\n * or not.\n */\nconst synthesizers = new Map<string, Promise<OfflineTts>>()\n\nfunction cacheKey(options: SynthesizerOptions): string {\n return JSON.stringify([\n options.modelDir,\n options.numThreads ?? null,\n options.provider ?? null,\n options.debug ?? null,\n ])\n}\n\nexport function loadSynthesizer(\n options: SynthesizerOptions\n): Promise<OfflineTts> {\n const key = cacheKey(options)\n const existing = synthesizers.get(key)\n if (existing) return existing\n\n // createAsync loads the graphs on a worker thread, so setting a call up does\n // not hold the event loop for the half second it takes\n const loading = Promise.resolve().then(() =>\n sherpa.OfflineTts.createAsync({\n model: {\n pocket: {\n ...resolveModelFiles(options.modelDir),\n voiceEmbeddingCacheCapacity: VOICE_CACHE_CAPACITY,\n },\n numThreads: options.numThreads ?? DEFAULT_NUM_THREADS,\n provider: options.provider ?? DEFAULT_PROVIDER,\n debug: options.debug ?? false,\n },\n // Sentences are already split upstream, one per synthesis\n maxNumSentences: 1,\n })\n )\n\n // A failed load must not poison the cache, the next call retries it\n loading.catch(() => synthesizers.delete(key))\n\n synthesizers.set(key, loading)\n return loading\n}\n\n/**\n * Reference voices read from disk, kept by path.\n *\n * The samples are handed to the addon on every sentence, which is where the\n * voice embedding gets cached, so what is saved here is only the file read.\n */\nconst voices = new Map<string, Wave>()\n\nexport function loadVoice(path: string): Wave {\n const existing = voices.get(path)\n if (existing) return existing\n\n if (!existsSync(path)) {\n throw new Error(`Pocket TTS reference voice not found: \"${path}\"`)\n }\n\n const wave = sherpa.readWave(path)\n voices.set(path, wave)\n return wave\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,gBAA2B;AAC3B,kBAAqB;AAarB,IAAM,UAAU;AAAA,EACd,QAAQ;AAAA,EACR,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,SAAS;AAAA,EACT,iBAAiB;AACnB;AAEA,IAAM,aAAa;AAAA,EACjB,WAAW;AAAA,EACX,iBAAiB;AACnB;AAEO,SAAS,kBACd,UACwD;AACxD,QAAM,QAAgC,CAAC;AACvC,QAAM,UAAoB,CAAC;AAE3B,aAAW,CAAC,KAAK,IAAI,KAAK,OAAO,QAAQ,OAAO,GAAG;AACjD,UAAM,QAAQ,CAAC,GAAG,IAAI,cAAc,GAAG,IAAI,OAAO,EAC/C,IAAI,CAAC,aAAS,kBAAK,UAAU,IAAI,CAAC,EAClC,KAAK,oBAAU;AAClB,QAAI,MAAO,OAAM,GAAG,IAAI;AAAA,QACnB,SAAQ,KAAK,GAAG,IAAI,iBAAiB,IAAI,OAAO;AAAA,EACvD;AAEA,aAAW,CAAC,KAAK,IAAI,KAAK,OAAO,QAAQ,UAAU,GAAG;AACpD,UAAM,WAAO,kBAAK,UAAU,IAAI;AAChC,YAAI,sBAAW,IAAI,EAAG,OAAM,GAAG,IAAI;AAAA,QAC9B,SAAQ,KAAK,IAAI;AAAA,EACxB;AAEA,MAAI,QAAQ,SAAS,GAAG;AACtB,UAAM,IAAI;AAAA,MACR,wCAAwC,QAAQ,MAAM,QAAQ,KAAK,IAAI,CAAC;AAAA,IAE1E;AAAA,EACF;AAEA,SAAO;AACT;AAUO,IAAM,iBAAiB;AAAA,EAC5B,MAAM;AAAA,EACN,OAAO;AAAA,EACP,cAAc;AAChB;AAKO,SAAS,iBACd,UACA,OACQ;AACR,QAAM,UAAU,eAAe,KAA2B;AAC1D,SAAO,cAAU,kBAAK,UAAU,OAAO,IAAI;AAC7C;;;AChFA,oBAA4D;AAC5D,IAAAA,2BAA8C;;;ACD9C,IAAAC,aAA2B;AAC3B,8BAAmD;AAUnD,IAAM,sBAAsB;AAC5B,IAAM,mBAAmB;AAEzB,IAAM,uBAAuB;AAW7B,IAAM,eAAe,oBAAI,IAAiC;AAE1D,SAAS,SAAS,SAAqC;AACrD,SAAO,KAAK,UAAU;AAAA,IACpB,QAAQ;AAAA,IACR,QAAQ,cAAc;AAAA,IACtB,QAAQ,YAAY;AAAA,IACpB,QAAQ,SAAS;AAAA,EACnB,CAAC;AACH;AAEO,SAAS,gBACd,SACqB;AACrB,QAAM,MAAM,SAAS,OAAO;AAC5B,QAAM,WAAW,aAAa,IAAI,GAAG;AACrC,MAAI,SAAU,QAAO;AAIrB,QAAM,UAAU,QAAQ,QAAQ,EAAE;AAAA,IAAK,MACrC,wBAAAC,QAAO,WAAW,YAAY;AAAA,MAC5B,OAAO;AAAA,QACL,QAAQ;AAAA,UACN,GAAG,kBAAkB,QAAQ,QAAQ;AAAA,UACrC,6BAA6B;AAAA,QAC/B;AAAA,QACA,YAAY,QAAQ,cAAc;AAAA,QAClC,UAAU,QAAQ,YAAY;AAAA,QAC9B,OAAO,QAAQ,SAAS;AAAA,MAC1B;AAAA;AAAA,MAEA,iBAAiB;AAAA,IACnB,CAAC;AAAA,EACH;AAGA,UAAQ,MAAM,MAAM,aAAa,OAAO,GAAG,CAAC;AAE5C,eAAa,IAAI,KAAK,OAAO;AAC7B,SAAO;AACT;AAQA,IAAM,SAAS,oBAAI,IAAkB;AAE9B,SAAS,UAAU,MAAoB;AAC5C,QAAM,WAAW,OAAO,IAAI,IAAI;AAChC,MAAI,SAAU,QAAO;AAErB,MAAI,KAAC,uBAAW,IAAI,GAAG;AACrB,UAAM,IAAI,MAAM,0CAA0C,IAAI,GAAG;AAAA,EACnE;AAEA,QAAM,OAAO,wBAAAA,QAAO,SAAS,IAAI;AACjC,SAAO,IAAI,MAAM,IAAI;AACrB,SAAO;AACT;;;AD7BA,IAAM,gBAAoC;AAC1C,IAAM,oBAAoB;AAC1B,IAAM,2BAA2B;AACjC,IAAM,qBAAqB;AAEpB,IAAM,YAAN,cAAwB,0BAAY;AAAA,EACzC,YAA6B,SAA2B;AACtD,UAAM;AADqB;AAI3B,sBAAkB,QAAQ,QAAQ;AAElC,oBAAgB,KAAK,sBAAsB,CAAC,EAAE,MAAM,CAAC,UAAU;AAC7D,cAAQ,MAAM,qCAAqC,KAAK;AAAA,IAC1D,CAAC;AAED,QAAI,QAAQ,WAAW,OAAO;AAC5B,WAAK,OAAO,EACT,KAAK,MAAM,KAAK,IAAI,iBAAiB,CAAC,EACtC,MAAM,MAAM;AAAA,MAAC,CAAC;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,MAAgB,WACd,MACA,QACoB;AACpB,UAAM,cAAc,MAAM,gBAAgB,KAAK,sBAAsB,CAAC;AAEtE,QAAI,OAAO,QAAS;AAIpB,UAAM,YAAY,IAAI;AAAA,MACpB,YAAY;AAAA,MACZ;AAAA,IACF;AAEA,UAAM,YAAY,cAAc;AAAA,MAC9B;AAAA;AAAA;AAAA,MAGA,sBAAsB;AAAA,MACtB,kBAAkB,KAAK,oBAAoB;AAAA,MAC3C,YAAY,CAAC,EAAE,QAAQ,MAAM;AAC3B,cAAM,QAAQ,UAAU,YAAQ,8BAAe,OAAO,CAAC;AAEvD,eAAO,KAAK,UAAU,KAAK,IAAI,IAAI;AAAA,MACrC;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAc,SAAwB;AACpC,UAAM,cAAc,MAAM,gBAAgB,KAAK,sBAAsB,CAAC;AACtE,UAAM,YAAY,cAAc;AAAA,MAC9B,MAAM;AAAA,MACN,kBAAkB,KAAK,oBAAoB;AAAA,IAC7C,CAAC;AAAA,EACH;AAAA,EAEQ,sBAAwC;AAC9C,UAAM,EAAE,QAAQ,IAAI;AACpB,UAAM,QAAQ;AAAA,MACZ,iBAAiB,QAAQ,UAAU,QAAQ,SAAS,aAAa;AAAA,IACnE;AAEA,WAAO,IAAI,yBAAAC,QAAO,iBAAiB;AAAA,MACjC,OAAO,QAAQ;AAAA,MACf,UAAU,QAAQ,YAAY;AAAA,MAC9B,gBAAgB,MAAM;AAAA,MACtB,qBAAqB,MAAM;AAAA,MAC3B,OAAO;AAAA,QACL,yBACE,QAAQ,mBAAmB;AAAA,QAC7B,GAAI,QAAQ,SAAS,SAAY,CAAC,IAAI,EAAE,MAAM,QAAQ,KAAK;AAAA,MAC7D;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,wBAA4C;AAClD,WAAO;AAAA,MACL,UAAU,KAAK,QAAQ;AAAA,MACvB,YAAY,KAAK,QAAQ;AAAA,MACzB,UAAU,KAAK,QAAQ;AAAA,IACzB;AAAA,EACF;AACF;","names":["import_sherpa_onnx_node","import_fs","sherpa","sherpa"]}
|
|
1
|
+
{"version":3,"sources":["../src/index.ts","../src/modelFiles.ts","../src/PocketTTS.ts","../src/synthesizer.ts"],"sourcesContent":["export * from './modelFiles'\nexport * from './PocketTTS'\n","import { existsSync } from 'fs'\nimport { join } from 'path'\nimport type { PocketModelConfig } from 'sherpa-onnx-node'\n\n/**\n * The files a Pocket TTS archive holds, resolved inside its folder.\n *\n * The model ships in two flavours, quantized and full precision, which name\n * their weights `lm_main.int8.onnx` and `lm_main.onnx`. Each part is looked up\n * in that order, so both archives work and a folder mixing them takes the\n * lighter file whenever there is one, which is what the quantized archive\n * itself does for its encoder and its text conditioner.\n */\n\nconst WEIGHTS = {\n lmFlow: 'lm_flow',\n lmMain: 'lm_main',\n encoder: 'encoder',\n decoder: 'decoder',\n textConditioner: 'text_conditioner',\n} as const\n\nconst JSON_FILES = {\n vocabJson: 'vocab.json',\n tokenScoresJson: 'token_scores.json',\n} as const\n\nexport function resolveModelFiles(\n modelDir: string\n): Omit<PocketModelConfig, 'voiceEmbeddingCacheCapacity'> {\n const files: Record<string, string> = {}\n const missing: string[] = []\n\n for (const [key, base] of Object.entries(WEIGHTS)) {\n const found = [`${base}.int8.onnx`, `${base}.onnx`]\n .map((name) => join(modelDir, name))\n .find(existsSync)\n if (found) files[key] = found\n else missing.push(`${base}.int8.onnx or ${base}.onnx`)\n }\n\n for (const [key, name] of Object.entries(JSON_FILES)) {\n const path = join(modelDir, name)\n if (existsSync(path)) files[key] = path\n else missing.push(name)\n }\n\n if (missing.length > 0) {\n throw new Error(\n `Pocket TTS model files not found in \"${modelDir}\": ${missing.join(', ')}. ` +\n 'Download the archive from https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models'\n )\n }\n\n return files as Omit<PocketModelConfig, 'voiceEmbeddingCacheCapacity'>\n}\n\n/**\n * The reference voices the archive ships in its `test_wavs` folder.\n *\n * `bria` is a woman reading calmly for forty seconds, `loona` is a one second\n * clip, which is the shortest a reference can reasonably be, and\n * `frenchAccent` is a French speaker, cloned into English words: the language\n * comes from the text, the timbre and the accent from the recording.\n */\nexport const BUNDLED_VOICES = {\n bria: 'test_wavs/bria.wav',\n loona: 'test_wavs/loona.wav',\n frenchAccent: 'test_wavs/sample_fr_hibiki_crepes.wav',\n} as const\n\nexport type PocketBundledVoice = keyof typeof BUNDLED_VOICES\n\n/** Turns a bundled voice name into a path, and leaves a path as it is. */\nexport function resolveVoicePath(\n modelDir: string,\n voice: PocketBundledVoice | (string & {})\n): string {\n const bundled = BUNDLED_VOICES[voice as PocketBundledVoice]\n return bundled ? join(modelDir, bundled) : voice\n}\n","import { float32ToPcm16, Pcm16Resampler, SentenceTTS } from '@micdrop/server'\nimport sherpa, { type GenerationConfig } from 'sherpa-onnx-node'\nimport {\n PocketBundledVoice,\n resolveModelFiles,\n resolveVoicePath,\n} from './modelFiles'\nimport { loadSynthesizer, loadVoice, SynthesizerOptions } from './synthesizer'\n\n/**\n * Local Pocket TTS text to speech, running in the Node process through the\n * sherpa-onnx addon and ONNX Runtime. Nothing leaves the machine and no server\n * has to be started next to it.\n *\n * Pocket TTS is a 100 million parameter model from Kyutai, built for the CPU,\n * that clones the voice of a few seconds of reference audio. It generates\n * several times faster than real time and hands its audio over while it is\n * still writing the rest, so the first words of a sentence reach the client in\n * about three hundred milliseconds.\n *\n * The weights are downloaded separately, see the README.\n *\n * @see https://github.com/kyutai-labs/pocket-tts\n * @see https://k2-fsa.github.io/sherpa/onnx/tts/pocket.html\n */\n\nexport interface PocketTTSOptions {\n /** Folder holding the model files, as extracted from the archive. */\n modelDir: string\n\n /**\n * Voice to clone: a bundled name (\"bria\"), or the path to a wav file\n * carrying a few seconds of the voice to speak with.\n */\n voice?: PocketBundledVoice | (string & {})\n\n /** Speech rate, 1 being the natural pace of the voice. */\n speed?: number\n\n /** Sampling steps per frame, higher sounding better and generating slower. */\n numSteps?: number\n\n /** Seconds of the reference audio the voice is built from. */\n referenceLength?: number\n\n /** Sampling seed, so the same sentence comes out the same way. */\n seed?: number\n\n /** Threads given to ONNX Runtime, 2 by default. */\n numThreads?: number\n\n /** Execution provider passed to ONNX Runtime, \"cpu\" by default. */\n provider?: 'cpu' | 'cuda' | 'coreml'\n\n /** Synthesizes one word at startup, on by default. */\n warmup?: boolean\n}\n\nconst DEFAULT_VOICE: PocketBundledVoice = 'bria'\nconst DEFAULT_NUM_STEPS = 2\nconst DEFAULT_REFERENCE_LENGTH = 12\nconst OUTPUT_SAMPLE_RATE = 16000 // Rate expected by the Micdrop client\n\nexport class PocketTTS extends SentenceTTS {\n constructor(private readonly options: PocketTTSOptions) {\n super()\n\n // Fail at startup rather than on the first sentence of the call\n resolveModelFiles(options.modelDir)\n\n loadSynthesizer(this.getSynthesizerOptions()).catch((error) => {\n console.error('[PocketTTS] Failed to load model:', error)\n })\n\n if (options.warmup !== false) {\n this.warmup()\n .then(() => this.log('Model warmed up'))\n .catch(() => {})\n }\n }\n\n protected async synthesize(\n text: string,\n signal: AbortSignal\n ): Promise<undefined> {\n const synthesizer = await loadSynthesizer(this.getSynthesizerOptions())\n // The utterance may have been cancelled while the model was loading\n if (signal.aborted) return\n\n // One resampler for the whole sentence: its chunks are a single continuous\n // stream, and the resampler carries its fractional position across them\n const resampler = new Pcm16Resampler(\n synthesizer.sampleRate,\n OUTPUT_SAMPLE_RATE\n )\n\n await synthesizer.generateAsync({\n text,\n // The samples of a chunk live in native memory for the length of the\n // callback, which is where they are copied into a PCM16 buffer\n enableExternalBuffer: true,\n generationConfig: this.getGenerationConfig(),\n onProgress: ({ samples }) => {\n const audio = resampler.process(float32ToPcm16(samples))\n // Stop generating what nobody is going to hear\n return this.emitAudio(audio) ? 1 : 0\n },\n })\n }\n\n /**\n * Speaks one short word to pay the first inference up front.\n *\n * It also builds the embedding of the voice, which the addon then keeps, so\n * the first sentence of the call only pays for its own generation.\n */\n private async warmup(): Promise<void> {\n const synthesizer = await loadSynthesizer(this.getSynthesizerOptions())\n await synthesizer.generateAsync({\n text: 'Hello.',\n generationConfig: this.getGenerationConfig(),\n })\n }\n\n private getGenerationConfig(): GenerationConfig {\n const { options } = this\n const voice = loadVoice(\n resolveVoicePath(options.modelDir, options.voice ?? DEFAULT_VOICE)\n )\n\n return new sherpa.GenerationConfig({\n speed: options.speed,\n numSteps: options.numSteps ?? DEFAULT_NUM_STEPS,\n referenceAudio: voice.samples,\n referenceSampleRate: voice.sampleRate,\n extra: {\n max_reference_audio_len:\n options.referenceLength ?? DEFAULT_REFERENCE_LENGTH,\n ...(options.seed === undefined ? {} : { seed: options.seed }),\n },\n })\n }\n\n private getSynthesizerOptions(): SynthesizerOptions {\n return {\n modelDir: this.options.modelDir,\n numThreads: this.options.numThreads,\n provider: this.options.provider,\n }\n }\n}\n","import { SharedInstances } from '@micdrop/server'\nimport { existsSync } from 'fs'\nimport sherpa, { type OfflineTts, type Wave } from 'sherpa-onnx-node'\nimport { resolveModelFiles } from './modelFiles'\n\nexport interface SynthesizerOptions {\n modelDir: string\n numThreads?: number\n provider?: string\n debug?: boolean\n}\n\nconst DEFAULT_NUM_THREADS = 2\nconst DEFAULT_PROVIDER = 'cpu'\n// Voices already turned into an embedding, kept by the addon between calls\nconst VOICE_CACHE_CAPACITY = 10\n\n/**\n * One instance per configuration, shared by every call.\n *\n * The weights weigh close to two hundred megabytes, so holding a second copy\n * for a second call would cost more memory than the language model it sits\n * next to. Inference is bound to a couple of threads anyway, and two calls\n * generating at once compete for the same cores whether they share the model\n * or not.\n */\nconst synthesizers = new SharedInstances<OfflineTts>()\n\nexport function loadSynthesizer(\n options: SynthesizerOptions\n): Promise<OfflineTts> {\n return synthesizers.load(\n [\n options.modelDir,\n options.numThreads ?? null,\n options.provider ?? null,\n options.debug ?? null,\n ],\n // createAsync loads the graphs on a worker thread, so setting a call up\n // does not hold the event loop for the half second it takes\n () =>\n sherpa.OfflineTts.createAsync({\n model: {\n pocket: {\n ...resolveModelFiles(options.modelDir),\n voiceEmbeddingCacheCapacity: VOICE_CACHE_CAPACITY,\n },\n numThreads: options.numThreads ?? DEFAULT_NUM_THREADS,\n provider: options.provider ?? DEFAULT_PROVIDER,\n debug: options.debug ?? false,\n },\n // Sentences are already split upstream, one per synthesis\n maxNumSentences: 1,\n })\n )\n}\n\n/**\n * Reference voices read from disk, kept by path.\n *\n * The samples are handed to the addon on every sentence, which is where the\n * voice embedding gets cached, so what is saved here is only the file read.\n */\nconst voices = new Map<string, Wave>()\n\nexport function loadVoice(path: string): Wave {\n const existing = voices.get(path)\n if (existing) return existing\n\n if (!existsSync(path)) {\n throw new Error(`Pocket TTS reference voice not found: \"${path}\"`)\n }\n\n const wave = sherpa.readWave(path)\n voices.set(path, wave)\n return wave\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,gBAA2B;AAC3B,kBAAqB;AAarB,IAAM,UAAU;AAAA,EACd,QAAQ;AAAA,EACR,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,SAAS;AAAA,EACT,iBAAiB;AACnB;AAEA,IAAM,aAAa;AAAA,EACjB,WAAW;AAAA,EACX,iBAAiB;AACnB;AAEO,SAAS,kBACd,UACwD;AACxD,QAAM,QAAgC,CAAC;AACvC,QAAM,UAAoB,CAAC;AAE3B,aAAW,CAAC,KAAK,IAAI,KAAK,OAAO,QAAQ,OAAO,GAAG;AACjD,UAAM,QAAQ,CAAC,GAAG,IAAI,cAAc,GAAG,IAAI,OAAO,EAC/C,IAAI,CAAC,aAAS,kBAAK,UAAU,IAAI,CAAC,EAClC,KAAK,oBAAU;AAClB,QAAI,MAAO,OAAM,GAAG,IAAI;AAAA,QACnB,SAAQ,KAAK,GAAG,IAAI,iBAAiB,IAAI,OAAO;AAAA,EACvD;AAEA,aAAW,CAAC,KAAK,IAAI,KAAK,OAAO,QAAQ,UAAU,GAAG;AACpD,UAAM,WAAO,kBAAK,UAAU,IAAI;AAChC,YAAI,sBAAW,IAAI,EAAG,OAAM,GAAG,IAAI;AAAA,QAC9B,SAAQ,KAAK,IAAI;AAAA,EACxB;AAEA,MAAI,QAAQ,SAAS,GAAG;AACtB,UAAM,IAAI;AAAA,MACR,wCAAwC,QAAQ,MAAM,QAAQ,KAAK,IAAI,CAAC;AAAA,IAE1E;AAAA,EACF;AAEA,SAAO;AACT;AAUO,IAAM,iBAAiB;AAAA,EAC5B,MAAM;AAAA,EACN,OAAO;AAAA,EACP,cAAc;AAChB;AAKO,SAAS,iBACd,UACA,OACQ;AACR,QAAM,UAAU,eAAe,KAA2B;AAC1D,SAAO,cAAU,kBAAK,UAAU,OAAO,IAAI;AAC7C;;;AChFA,IAAAA,iBAA4D;AAC5D,IAAAC,2BAA8C;;;ACD9C,oBAAgC;AAChC,IAAAC,aAA2B;AAC3B,8BAAmD;AAUnD,IAAM,sBAAsB;AAC5B,IAAM,mBAAmB;AAEzB,IAAM,uBAAuB;AAW7B,IAAM,eAAe,IAAI,8BAA4B;AAE9C,SAAS,gBACd,SACqB;AACrB,SAAO,aAAa;AAAA,IAClB;AAAA,MACE,QAAQ;AAAA,MACR,QAAQ,cAAc;AAAA,MACtB,QAAQ,YAAY;AAAA,MACpB,QAAQ,SAAS;AAAA,IACnB;AAAA;AAAA;AAAA,IAGA,MACE,wBAAAC,QAAO,WAAW,YAAY;AAAA,MAC5B,OAAO;AAAA,QACL,QAAQ;AAAA,UACN,GAAG,kBAAkB,QAAQ,QAAQ;AAAA,UACrC,6BAA6B;AAAA,QAC/B;AAAA,QACA,YAAY,QAAQ,cAAc;AAAA,QAClC,UAAU,QAAQ,YAAY;AAAA,QAC9B,OAAO,QAAQ,SAAS;AAAA,MAC1B;AAAA;AAAA,MAEA,iBAAiB;AAAA,IACnB,CAAC;AAAA,EACL;AACF;AAQA,IAAM,SAAS,oBAAI,IAAkB;AAE9B,SAAS,UAAU,MAAoB;AAC5C,QAAM,WAAW,OAAO,IAAI,IAAI;AAChC,MAAI,SAAU,QAAO;AAErB,MAAI,KAAC,uBAAW,IAAI,GAAG;AACrB,UAAM,IAAI,MAAM,0CAA0C,IAAI,GAAG;AAAA,EACnE;AAEA,QAAM,OAAO,wBAAAA,QAAO,SAAS,IAAI;AACjC,SAAO,IAAI,MAAM,IAAI;AACrB,SAAO;AACT;;;ADlBA,IAAM,gBAAoC;AAC1C,IAAM,oBAAoB;AAC1B,IAAM,2BAA2B;AACjC,IAAM,qBAAqB;AAEpB,IAAM,YAAN,cAAwB,2BAAY;AAAA,EACzC,YAA6B,SAA2B;AACtD,UAAM;AADqB;AAI3B,sBAAkB,QAAQ,QAAQ;AAElC,oBAAgB,KAAK,sBAAsB,CAAC,EAAE,MAAM,CAAC,UAAU;AAC7D,cAAQ,MAAM,qCAAqC,KAAK;AAAA,IAC1D,CAAC;AAED,QAAI,QAAQ,WAAW,OAAO;AAC5B,WAAK,OAAO,EACT,KAAK,MAAM,KAAK,IAAI,iBAAiB,CAAC,EACtC,MAAM,MAAM;AAAA,MAAC,CAAC;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,MAAgB,WACd,MACA,QACoB;AACpB,UAAM,cAAc,MAAM,gBAAgB,KAAK,sBAAsB,CAAC;AAEtE,QAAI,OAAO,QAAS;AAIpB,UAAM,YAAY,IAAI;AAAA,MACpB,YAAY;AAAA,MACZ;AAAA,IACF;AAEA,UAAM,YAAY,cAAc;AAAA,MAC9B;AAAA;AAAA;AAAA,MAGA,sBAAsB;AAAA,MACtB,kBAAkB,KAAK,oBAAoB;AAAA,MAC3C,YAAY,CAAC,EAAE,QAAQ,MAAM;AAC3B,cAAM,QAAQ,UAAU,YAAQ,+BAAe,OAAO,CAAC;AAEvD,eAAO,KAAK,UAAU,KAAK,IAAI,IAAI;AAAA,MACrC;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAc,SAAwB;AACpC,UAAM,cAAc,MAAM,gBAAgB,KAAK,sBAAsB,CAAC;AACtE,UAAM,YAAY,cAAc;AAAA,MAC9B,MAAM;AAAA,MACN,kBAAkB,KAAK,oBAAoB;AAAA,IAC7C,CAAC;AAAA,EACH;AAAA,EAEQ,sBAAwC;AAC9C,UAAM,EAAE,QAAQ,IAAI;AACpB,UAAM,QAAQ;AAAA,MACZ,iBAAiB,QAAQ,UAAU,QAAQ,SAAS,aAAa;AAAA,IACnE;AAEA,WAAO,IAAI,yBAAAC,QAAO,iBAAiB;AAAA,MACjC,OAAO,QAAQ;AAAA,MACf,UAAU,QAAQ,YAAY;AAAA,MAC9B,gBAAgB,MAAM;AAAA,MACtB,qBAAqB,MAAM;AAAA,MAC3B,OAAO;AAAA,QACL,yBACE,QAAQ,mBAAmB;AAAA,QAC7B,GAAI,QAAQ,SAAS,SAAY,CAAC,IAAI,EAAE,MAAM,QAAQ,KAAK;AAAA,MAC7D;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,wBAA4C;AAClD,WAAO;AAAA,MACL,UAAU,KAAK,QAAQ;AAAA,MACvB,YAAY,KAAK,QAAQ;AAAA,MACzB,UAAU,KAAK,QAAQ;AAAA,IACzB;AAAA,EACF;AACF;","names":["import_server","import_sherpa_onnx_node","import_fs","sherpa","sherpa"]}
|
package/dist/index.mjs
CHANGED
|
@@ -47,25 +47,23 @@ import { float32ToPcm16, Pcm16Resampler, SentenceTTS } from "@micdrop/server";
|
|
|
47
47
|
import sherpa2 from "sherpa-onnx-node";
|
|
48
48
|
|
|
49
49
|
// src/synthesizer.ts
|
|
50
|
+
import { SharedInstances } from "@micdrop/server";
|
|
50
51
|
import { existsSync as existsSync2 } from "fs";
|
|
51
52
|
import sherpa from "sherpa-onnx-node";
|
|
52
53
|
var DEFAULT_NUM_THREADS = 2;
|
|
53
54
|
var DEFAULT_PROVIDER = "cpu";
|
|
54
55
|
var VOICE_CACHE_CAPACITY = 10;
|
|
55
|
-
var synthesizers =
|
|
56
|
-
function cacheKey(options) {
|
|
57
|
-
return JSON.stringify([
|
|
58
|
-
options.modelDir,
|
|
59
|
-
options.numThreads ?? null,
|
|
60
|
-
options.provider ?? null,
|
|
61
|
-
options.debug ?? null
|
|
62
|
-
]);
|
|
63
|
-
}
|
|
56
|
+
var synthesizers = new SharedInstances();
|
|
64
57
|
function loadSynthesizer(options) {
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
58
|
+
return synthesizers.load(
|
|
59
|
+
[
|
|
60
|
+
options.modelDir,
|
|
61
|
+
options.numThreads ?? null,
|
|
62
|
+
options.provider ?? null,
|
|
63
|
+
options.debug ?? null
|
|
64
|
+
],
|
|
65
|
+
// createAsync loads the graphs on a worker thread, so setting a call up
|
|
66
|
+
// does not hold the event loop for the half second it takes
|
|
69
67
|
() => sherpa.OfflineTts.createAsync({
|
|
70
68
|
model: {
|
|
71
69
|
pocket: {
|
|
@@ -80,9 +78,6 @@ function loadSynthesizer(options) {
|
|
|
80
78
|
maxNumSentences: 1
|
|
81
79
|
})
|
|
82
80
|
);
|
|
83
|
-
loading.catch(() => synthesizers.delete(key));
|
|
84
|
-
synthesizers.set(key, loading);
|
|
85
|
-
return loading;
|
|
86
81
|
}
|
|
87
82
|
var voices = /* @__PURE__ */ new Map();
|
|
88
83
|
function loadVoice(path) {
|
package/dist/index.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/modelFiles.ts","../src/PocketTTS.ts","../src/synthesizer.ts"],"sourcesContent":["import { existsSync } from 'fs'\nimport { join } from 'path'\nimport type { PocketModelConfig } from 'sherpa-onnx-node'\n\n/**\n * The files a Pocket TTS archive holds, resolved inside its folder.\n *\n * The model ships in two flavours, quantized and full precision, which name\n * their weights `lm_main.int8.onnx` and `lm_main.onnx`. Each part is looked up\n * in that order, so both archives work and a folder mixing them takes the\n * lighter file whenever there is one, which is what the quantized archive\n * itself does for its encoder and its text conditioner.\n */\n\nconst WEIGHTS = {\n lmFlow: 'lm_flow',\n lmMain: 'lm_main',\n encoder: 'encoder',\n decoder: 'decoder',\n textConditioner: 'text_conditioner',\n} as const\n\nconst JSON_FILES = {\n vocabJson: 'vocab.json',\n tokenScoresJson: 'token_scores.json',\n} as const\n\nexport function resolveModelFiles(\n modelDir: string\n): Omit<PocketModelConfig, 'voiceEmbeddingCacheCapacity'> {\n const files: Record<string, string> = {}\n const missing: string[] = []\n\n for (const [key, base] of Object.entries(WEIGHTS)) {\n const found = [`${base}.int8.onnx`, `${base}.onnx`]\n .map((name) => join(modelDir, name))\n .find(existsSync)\n if (found) files[key] = found\n else missing.push(`${base}.int8.onnx or ${base}.onnx`)\n }\n\n for (const [key, name] of Object.entries(JSON_FILES)) {\n const path = join(modelDir, name)\n if (existsSync(path)) files[key] = path\n else missing.push(name)\n }\n\n if (missing.length > 0) {\n throw new Error(\n `Pocket TTS model files not found in \"${modelDir}\": ${missing.join(', ')}. ` +\n 'Download the archive from https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models'\n )\n }\n\n return files as Omit<PocketModelConfig, 'voiceEmbeddingCacheCapacity'>\n}\n\n/**\n * The reference voices the archive ships in its `test_wavs` folder.\n *\n * `bria` is a woman reading calmly for forty seconds, `loona` is a one second\n * clip, which is the shortest a reference can reasonably be, and\n * `frenchAccent` is a French speaker, cloned into English words: the language\n * comes from the text, the timbre and the accent from the recording.\n */\nexport const BUNDLED_VOICES = {\n bria: 'test_wavs/bria.wav',\n loona: 'test_wavs/loona.wav',\n frenchAccent: 'test_wavs/sample_fr_hibiki_crepes.wav',\n} as const\n\nexport type PocketBundledVoice = keyof typeof BUNDLED_VOICES\n\n/** Turns a bundled voice name into a path, and leaves a path as it is. */\nexport function resolveVoicePath(\n modelDir: string,\n voice: PocketBundledVoice | (string & {})\n): string {\n const bundled = BUNDLED_VOICES[voice as PocketBundledVoice]\n return bundled ? join(modelDir, bundled) : voice\n}\n","import { float32ToPcm16, Pcm16Resampler, SentenceTTS } from '@micdrop/server'\nimport sherpa, { type GenerationConfig } from 'sherpa-onnx-node'\nimport {\n PocketBundledVoice,\n resolveModelFiles,\n resolveVoicePath,\n} from './modelFiles'\nimport { loadSynthesizer, loadVoice, SynthesizerOptions } from './synthesizer'\n\n/**\n * Local Pocket TTS text to speech, running in the Node process through the\n * sherpa-onnx addon and ONNX Runtime. Nothing leaves the machine and no server\n * has to be started next to it.\n *\n * Pocket TTS is a 100 million parameter model from Kyutai, built for the CPU,\n * that clones the voice of a few seconds of reference audio. It generates\n * several times faster than real time and hands its audio over while it is\n * still writing the rest, so the first words of a sentence reach the client in\n * about three hundred milliseconds.\n *\n * The weights are downloaded separately, see the README.\n *\n * @see https://github.com/kyutai-labs/pocket-tts\n * @see https://k2-fsa.github.io/sherpa/onnx/tts/pocket.html\n */\n\nexport interface PocketTTSOptions {\n /** Folder holding the model files, as extracted from the archive. */\n modelDir: string\n\n /**\n * Voice to clone: a bundled name (\"bria\"), or the path to a wav file\n * carrying a few seconds of the voice to speak with.\n */\n voice?: PocketBundledVoice | (string & {})\n\n /** Speech rate, 1 being the natural pace of the voice. */\n speed?: number\n\n /** Sampling steps per frame, higher sounding better and generating slower. */\n numSteps?: number\n\n /** Seconds of the reference audio the voice is built from. */\n referenceLength?: number\n\n /** Sampling seed, so the same sentence comes out the same way. */\n seed?: number\n\n /** Threads given to ONNX Runtime, 2 by default. */\n numThreads?: number\n\n /** Execution provider passed to ONNX Runtime, \"cpu\" by default. */\n provider?: 'cpu' | 'cuda' | 'coreml'\n\n /** Synthesizes one word at startup, on by default. */\n warmup?: boolean\n}\n\nconst DEFAULT_VOICE: PocketBundledVoice = 'bria'\nconst DEFAULT_NUM_STEPS = 2\nconst DEFAULT_REFERENCE_LENGTH = 12\nconst OUTPUT_SAMPLE_RATE = 16000 // Rate expected by the Micdrop client\n\nexport class PocketTTS extends SentenceTTS {\n constructor(private readonly options: PocketTTSOptions) {\n super()\n\n // Fail at startup rather than on the first sentence of the call\n resolveModelFiles(options.modelDir)\n\n loadSynthesizer(this.getSynthesizerOptions()).catch((error) => {\n console.error('[PocketTTS] Failed to load model:', error)\n })\n\n if (options.warmup !== false) {\n this.warmup()\n .then(() => this.log('Model warmed up'))\n .catch(() => {})\n }\n }\n\n protected async synthesize(\n text: string,\n signal: AbortSignal\n ): Promise<undefined> {\n const synthesizer = await loadSynthesizer(this.getSynthesizerOptions())\n // The utterance may have been cancelled while the model was loading\n if (signal.aborted) return\n\n // One resampler for the whole sentence: its chunks are a single continuous\n // stream, and the resampler carries its fractional position across them\n const resampler = new Pcm16Resampler(\n synthesizer.sampleRate,\n OUTPUT_SAMPLE_RATE\n )\n\n await synthesizer.generateAsync({\n text,\n // The samples of a chunk live in native memory for the length of the\n // callback, which is where they are copied into a PCM16 buffer\n enableExternalBuffer: true,\n generationConfig: this.getGenerationConfig(),\n onProgress: ({ samples }) => {\n const audio = resampler.process(float32ToPcm16(samples))\n // Stop generating what nobody is going to hear\n return this.emitAudio(audio) ? 1 : 0\n },\n })\n }\n\n /**\n * Speaks one short word to pay the first inference up front.\n *\n * It also builds the embedding of the voice, which the addon then keeps, so\n * the first sentence of the call only pays for its own generation.\n */\n private async warmup(): Promise<void> {\n const synthesizer = await loadSynthesizer(this.getSynthesizerOptions())\n await synthesizer.generateAsync({\n text: 'Hello.',\n generationConfig: this.getGenerationConfig(),\n })\n }\n\n private getGenerationConfig(): GenerationConfig {\n const { options } = this\n const voice = loadVoice(\n resolveVoicePath(options.modelDir, options.voice ?? DEFAULT_VOICE)\n )\n\n return new sherpa.GenerationConfig({\n speed: options.speed,\n numSteps: options.numSteps ?? DEFAULT_NUM_STEPS,\n referenceAudio: voice.samples,\n referenceSampleRate: voice.sampleRate,\n extra: {\n max_reference_audio_len:\n options.referenceLength ?? DEFAULT_REFERENCE_LENGTH,\n ...(options.seed === undefined ? {} : { seed: options.seed }),\n },\n })\n }\n\n private getSynthesizerOptions(): SynthesizerOptions {\n return {\n modelDir: this.options.modelDir,\n numThreads: this.options.numThreads,\n provider: this.options.provider,\n }\n }\n}\n","import { existsSync } from 'fs'\nimport sherpa, { type OfflineTts, type Wave } from 'sherpa-onnx-node'\nimport { resolveModelFiles } from './modelFiles'\n\nexport interface SynthesizerOptions {\n modelDir: string\n numThreads?: number\n provider?: string\n debug?: boolean\n}\n\nconst DEFAULT_NUM_THREADS = 2\nconst DEFAULT_PROVIDER = 'cpu'\n// Voices already turned into an embedding, kept by the addon between calls\nconst VOICE_CACHE_CAPACITY = 10\n\n/**\n * One instance per configuration, shared by every call.\n *\n * The weights weigh close to two hundred megabytes, so holding a second copy\n * for a second call would cost more memory than the language model it sits\n * next to. Inference is bound to a couple of threads anyway, and two calls\n * generating at once compete for the same cores whether they share the model\n * or not.\n */\nconst synthesizers = new Map<string, Promise<OfflineTts>>()\n\nfunction cacheKey(options: SynthesizerOptions): string {\n return JSON.stringify([\n options.modelDir,\n options.numThreads ?? null,\n options.provider ?? null,\n options.debug ?? null,\n ])\n}\n\nexport function loadSynthesizer(\n options: SynthesizerOptions\n): Promise<OfflineTts> {\n const key = cacheKey(options)\n const existing = synthesizers.get(key)\n if (existing) return existing\n\n // createAsync loads the graphs on a worker thread, so setting a call up does\n // not hold the event loop for the half second it takes\n const loading = Promise.resolve().then(() =>\n sherpa.OfflineTts.createAsync({\n model: {\n pocket: {\n ...resolveModelFiles(options.modelDir),\n voiceEmbeddingCacheCapacity: VOICE_CACHE_CAPACITY,\n },\n numThreads: options.numThreads ?? DEFAULT_NUM_THREADS,\n provider: options.provider ?? DEFAULT_PROVIDER,\n debug: options.debug ?? false,\n },\n // Sentences are already split upstream, one per synthesis\n maxNumSentences: 1,\n })\n )\n\n // A failed load must not poison the cache, the next call retries it\n loading.catch(() => synthesizers.delete(key))\n\n synthesizers.set(key, loading)\n return loading\n}\n\n/**\n * Reference voices read from disk, kept by path.\n *\n * The samples are handed to the addon on every sentence, which is where the\n * voice embedding gets cached, so what is saved here is only the file read.\n */\nconst voices = new Map<string, Wave>()\n\nexport function loadVoice(path: string): Wave {\n const existing = voices.get(path)\n if (existing) return existing\n\n if (!existsSync(path)) {\n throw new Error(`Pocket TTS reference voice not found: \"${path}\"`)\n }\n\n const wave = sherpa.readWave(path)\n voices.set(path, wave)\n return wave\n}\n"],"mappings":";AAAA,SAAS,kBAAkB;AAC3B,SAAS,YAAY;AAarB,IAAM,UAAU;AAAA,EACd,QAAQ;AAAA,EACR,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,SAAS;AAAA,EACT,iBAAiB;AACnB;AAEA,IAAM,aAAa;AAAA,EACjB,WAAW;AAAA,EACX,iBAAiB;AACnB;AAEO,SAAS,kBACd,UACwD;AACxD,QAAM,QAAgC,CAAC;AACvC,QAAM,UAAoB,CAAC;AAE3B,aAAW,CAAC,KAAK,IAAI,KAAK,OAAO,QAAQ,OAAO,GAAG;AACjD,UAAM,QAAQ,CAAC,GAAG,IAAI,cAAc,GAAG,IAAI,OAAO,EAC/C,IAAI,CAAC,SAAS,KAAK,UAAU,IAAI,CAAC,EAClC,KAAK,UAAU;AAClB,QAAI,MAAO,OAAM,GAAG,IAAI;AAAA,QACnB,SAAQ,KAAK,GAAG,IAAI,iBAAiB,IAAI,OAAO;AAAA,EACvD;AAEA,aAAW,CAAC,KAAK,IAAI,KAAK,OAAO,QAAQ,UAAU,GAAG;AACpD,UAAM,OAAO,KAAK,UAAU,IAAI;AAChC,QAAI,WAAW,IAAI,EAAG,OAAM,GAAG,IAAI;AAAA,QAC9B,SAAQ,KAAK,IAAI;AAAA,EACxB;AAEA,MAAI,QAAQ,SAAS,GAAG;AACtB,UAAM,IAAI;AAAA,MACR,wCAAwC,QAAQ,MAAM,QAAQ,KAAK,IAAI,CAAC;AAAA,IAE1E;AAAA,EACF;AAEA,SAAO;AACT;AAUO,IAAM,iBAAiB;AAAA,EAC5B,MAAM;AAAA,EACN,OAAO;AAAA,EACP,cAAc;AAChB;AAKO,SAAS,iBACd,UACA,OACQ;AACR,QAAM,UAAU,eAAe,KAA2B;AAC1D,SAAO,UAAU,KAAK,UAAU,OAAO,IAAI;AAC7C;;;AChFA,SAAS,gBAAgB,gBAAgB,mBAAmB;AAC5D,OAAOA,aAAuC;;;ACD9C,SAAS,cAAAC,mBAAkB;AAC3B,OAAO,YAA4C;AAUnD,IAAM,sBAAsB;AAC5B,IAAM,mBAAmB;AAEzB,IAAM,uBAAuB;AAW7B,IAAM,eAAe,oBAAI,IAAiC;AAE1D,SAAS,SAAS,SAAqC;AACrD,SAAO,KAAK,UAAU;AAAA,IACpB,QAAQ;AAAA,IACR,QAAQ,cAAc;AAAA,IACtB,QAAQ,YAAY;AAAA,IACpB,QAAQ,SAAS;AAAA,EACnB,CAAC;AACH;AAEO,SAAS,gBACd,SACqB;AACrB,QAAM,MAAM,SAAS,OAAO;AAC5B,QAAM,WAAW,aAAa,IAAI,GAAG;AACrC,MAAI,SAAU,QAAO;AAIrB,QAAM,UAAU,QAAQ,QAAQ,EAAE;AAAA,IAAK,MACrC,OAAO,WAAW,YAAY;AAAA,MAC5B,OAAO;AAAA,QACL,QAAQ;AAAA,UACN,GAAG,kBAAkB,QAAQ,QAAQ;AAAA,UACrC,6BAA6B;AAAA,QAC/B;AAAA,QACA,YAAY,QAAQ,cAAc;AAAA,QAClC,UAAU,QAAQ,YAAY;AAAA,QAC9B,OAAO,QAAQ,SAAS;AAAA,MAC1B;AAAA;AAAA,MAEA,iBAAiB;AAAA,IACnB,CAAC;AAAA,EACH;AAGA,UAAQ,MAAM,MAAM,aAAa,OAAO,GAAG,CAAC;AAE5C,eAAa,IAAI,KAAK,OAAO;AAC7B,SAAO;AACT;AAQA,IAAM,SAAS,oBAAI,IAAkB;AAE9B,SAAS,UAAU,MAAoB;AAC5C,QAAM,WAAW,OAAO,IAAI,IAAI;AAChC,MAAI,SAAU,QAAO;AAErB,MAAI,CAACC,YAAW,IAAI,GAAG;AACrB,UAAM,IAAI,MAAM,0CAA0C,IAAI,GAAG;AAAA,EACnE;AAEA,QAAM,OAAO,OAAO,SAAS,IAAI;AACjC,SAAO,IAAI,MAAM,IAAI;AACrB,SAAO;AACT;;;AD7BA,IAAM,gBAAoC;AAC1C,IAAM,oBAAoB;AAC1B,IAAM,2BAA2B;AACjC,IAAM,qBAAqB;AAEpB,IAAM,YAAN,cAAwB,YAAY;AAAA,EACzC,YAA6B,SAA2B;AACtD,UAAM;AADqB;AAI3B,sBAAkB,QAAQ,QAAQ;AAElC,oBAAgB,KAAK,sBAAsB,CAAC,EAAE,MAAM,CAAC,UAAU;AAC7D,cAAQ,MAAM,qCAAqC,KAAK;AAAA,IAC1D,CAAC;AAED,QAAI,QAAQ,WAAW,OAAO;AAC5B,WAAK,OAAO,EACT,KAAK,MAAM,KAAK,IAAI,iBAAiB,CAAC,EACtC,MAAM,MAAM;AAAA,MAAC,CAAC;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,MAAgB,WACd,MACA,QACoB;AACpB,UAAM,cAAc,MAAM,gBAAgB,KAAK,sBAAsB,CAAC;AAEtE,QAAI,OAAO,QAAS;AAIpB,UAAM,YAAY,IAAI;AAAA,MACpB,YAAY;AAAA,MACZ;AAAA,IACF;AAEA,UAAM,YAAY,cAAc;AAAA,MAC9B;AAAA;AAAA;AAAA,MAGA,sBAAsB;AAAA,MACtB,kBAAkB,KAAK,oBAAoB;AAAA,MAC3C,YAAY,CAAC,EAAE,QAAQ,MAAM;AAC3B,cAAM,QAAQ,UAAU,QAAQ,eAAe,OAAO,CAAC;AAEvD,eAAO,KAAK,UAAU,KAAK,IAAI,IAAI;AAAA,MACrC;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAc,SAAwB;AACpC,UAAM,cAAc,MAAM,gBAAgB,KAAK,sBAAsB,CAAC;AACtE,UAAM,YAAY,cAAc;AAAA,MAC9B,MAAM;AAAA,MACN,kBAAkB,KAAK,oBAAoB;AAAA,IAC7C,CAAC;AAAA,EACH;AAAA,EAEQ,sBAAwC;AAC9C,UAAM,EAAE,QAAQ,IAAI;AACpB,UAAM,QAAQ;AAAA,MACZ,iBAAiB,QAAQ,UAAU,QAAQ,SAAS,aAAa;AAAA,IACnE;AAEA,WAAO,IAAIC,QAAO,iBAAiB;AAAA,MACjC,OAAO,QAAQ;AAAA,MACf,UAAU,QAAQ,YAAY;AAAA,MAC9B,gBAAgB,MAAM;AAAA,MACtB,qBAAqB,MAAM;AAAA,MAC3B,OAAO;AAAA,QACL,yBACE,QAAQ,mBAAmB;AAAA,QAC7B,GAAI,QAAQ,SAAS,SAAY,CAAC,IAAI,EAAE,MAAM,QAAQ,KAAK;AAAA,MAC7D;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,wBAA4C;AAClD,WAAO;AAAA,MACL,UAAU,KAAK,QAAQ;AAAA,MACvB,YAAY,KAAK,QAAQ;AAAA,MACzB,UAAU,KAAK,QAAQ;AAAA,IACzB;AAAA,EACF;AACF;","names":["sherpa","existsSync","existsSync","sherpa"]}
|
|
1
|
+
{"version":3,"sources":["../src/modelFiles.ts","../src/PocketTTS.ts","../src/synthesizer.ts"],"sourcesContent":["import { existsSync } from 'fs'\nimport { join } from 'path'\nimport type { PocketModelConfig } from 'sherpa-onnx-node'\n\n/**\n * The files a Pocket TTS archive holds, resolved inside its folder.\n *\n * The model ships in two flavours, quantized and full precision, which name\n * their weights `lm_main.int8.onnx` and `lm_main.onnx`. Each part is looked up\n * in that order, so both archives work and a folder mixing them takes the\n * lighter file whenever there is one, which is what the quantized archive\n * itself does for its encoder and its text conditioner.\n */\n\nconst WEIGHTS = {\n lmFlow: 'lm_flow',\n lmMain: 'lm_main',\n encoder: 'encoder',\n decoder: 'decoder',\n textConditioner: 'text_conditioner',\n} as const\n\nconst JSON_FILES = {\n vocabJson: 'vocab.json',\n tokenScoresJson: 'token_scores.json',\n} as const\n\nexport function resolveModelFiles(\n modelDir: string\n): Omit<PocketModelConfig, 'voiceEmbeddingCacheCapacity'> {\n const files: Record<string, string> = {}\n const missing: string[] = []\n\n for (const [key, base] of Object.entries(WEIGHTS)) {\n const found = [`${base}.int8.onnx`, `${base}.onnx`]\n .map((name) => join(modelDir, name))\n .find(existsSync)\n if (found) files[key] = found\n else missing.push(`${base}.int8.onnx or ${base}.onnx`)\n }\n\n for (const [key, name] of Object.entries(JSON_FILES)) {\n const path = join(modelDir, name)\n if (existsSync(path)) files[key] = path\n else missing.push(name)\n }\n\n if (missing.length > 0) {\n throw new Error(\n `Pocket TTS model files not found in \"${modelDir}\": ${missing.join(', ')}. ` +\n 'Download the archive from https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models'\n )\n }\n\n return files as Omit<PocketModelConfig, 'voiceEmbeddingCacheCapacity'>\n}\n\n/**\n * The reference voices the archive ships in its `test_wavs` folder.\n *\n * `bria` is a woman reading calmly for forty seconds, `loona` is a one second\n * clip, which is the shortest a reference can reasonably be, and\n * `frenchAccent` is a French speaker, cloned into English words: the language\n * comes from the text, the timbre and the accent from the recording.\n */\nexport const BUNDLED_VOICES = {\n bria: 'test_wavs/bria.wav',\n loona: 'test_wavs/loona.wav',\n frenchAccent: 'test_wavs/sample_fr_hibiki_crepes.wav',\n} as const\n\nexport type PocketBundledVoice = keyof typeof BUNDLED_VOICES\n\n/** Turns a bundled voice name into a path, and leaves a path as it is. */\nexport function resolveVoicePath(\n modelDir: string,\n voice: PocketBundledVoice | (string & {})\n): string {\n const bundled = BUNDLED_VOICES[voice as PocketBundledVoice]\n return bundled ? join(modelDir, bundled) : voice\n}\n","import { float32ToPcm16, Pcm16Resampler, SentenceTTS } from '@micdrop/server'\nimport sherpa, { type GenerationConfig } from 'sherpa-onnx-node'\nimport {\n PocketBundledVoice,\n resolveModelFiles,\n resolveVoicePath,\n} from './modelFiles'\nimport { loadSynthesizer, loadVoice, SynthesizerOptions } from './synthesizer'\n\n/**\n * Local Pocket TTS text to speech, running in the Node process through the\n * sherpa-onnx addon and ONNX Runtime. Nothing leaves the machine and no server\n * has to be started next to it.\n *\n * Pocket TTS is a 100 million parameter model from Kyutai, built for the CPU,\n * that clones the voice of a few seconds of reference audio. It generates\n * several times faster than real time and hands its audio over while it is\n * still writing the rest, so the first words of a sentence reach the client in\n * about three hundred milliseconds.\n *\n * The weights are downloaded separately, see the README.\n *\n * @see https://github.com/kyutai-labs/pocket-tts\n * @see https://k2-fsa.github.io/sherpa/onnx/tts/pocket.html\n */\n\nexport interface PocketTTSOptions {\n /** Folder holding the model files, as extracted from the archive. */\n modelDir: string\n\n /**\n * Voice to clone: a bundled name (\"bria\"), or the path to a wav file\n * carrying a few seconds of the voice to speak with.\n */\n voice?: PocketBundledVoice | (string & {})\n\n /** Speech rate, 1 being the natural pace of the voice. */\n speed?: number\n\n /** Sampling steps per frame, higher sounding better and generating slower. */\n numSteps?: number\n\n /** Seconds of the reference audio the voice is built from. */\n referenceLength?: number\n\n /** Sampling seed, so the same sentence comes out the same way. */\n seed?: number\n\n /** Threads given to ONNX Runtime, 2 by default. */\n numThreads?: number\n\n /** Execution provider passed to ONNX Runtime, \"cpu\" by default. */\n provider?: 'cpu' | 'cuda' | 'coreml'\n\n /** Synthesizes one word at startup, on by default. */\n warmup?: boolean\n}\n\nconst DEFAULT_VOICE: PocketBundledVoice = 'bria'\nconst DEFAULT_NUM_STEPS = 2\nconst DEFAULT_REFERENCE_LENGTH = 12\nconst OUTPUT_SAMPLE_RATE = 16000 // Rate expected by the Micdrop client\n\nexport class PocketTTS extends SentenceTTS {\n constructor(private readonly options: PocketTTSOptions) {\n super()\n\n // Fail at startup rather than on the first sentence of the call\n resolveModelFiles(options.modelDir)\n\n loadSynthesizer(this.getSynthesizerOptions()).catch((error) => {\n console.error('[PocketTTS] Failed to load model:', error)\n })\n\n if (options.warmup !== false) {\n this.warmup()\n .then(() => this.log('Model warmed up'))\n .catch(() => {})\n }\n }\n\n protected async synthesize(\n text: string,\n signal: AbortSignal\n ): Promise<undefined> {\n const synthesizer = await loadSynthesizer(this.getSynthesizerOptions())\n // The utterance may have been cancelled while the model was loading\n if (signal.aborted) return\n\n // One resampler for the whole sentence: its chunks are a single continuous\n // stream, and the resampler carries its fractional position across them\n const resampler = new Pcm16Resampler(\n synthesizer.sampleRate,\n OUTPUT_SAMPLE_RATE\n )\n\n await synthesizer.generateAsync({\n text,\n // The samples of a chunk live in native memory for the length of the\n // callback, which is where they are copied into a PCM16 buffer\n enableExternalBuffer: true,\n generationConfig: this.getGenerationConfig(),\n onProgress: ({ samples }) => {\n const audio = resampler.process(float32ToPcm16(samples))\n // Stop generating what nobody is going to hear\n return this.emitAudio(audio) ? 1 : 0\n },\n })\n }\n\n /**\n * Speaks one short word to pay the first inference up front.\n *\n * It also builds the embedding of the voice, which the addon then keeps, so\n * the first sentence of the call only pays for its own generation.\n */\n private async warmup(): Promise<void> {\n const synthesizer = await loadSynthesizer(this.getSynthesizerOptions())\n await synthesizer.generateAsync({\n text: 'Hello.',\n generationConfig: this.getGenerationConfig(),\n })\n }\n\n private getGenerationConfig(): GenerationConfig {\n const { options } = this\n const voice = loadVoice(\n resolveVoicePath(options.modelDir, options.voice ?? DEFAULT_VOICE)\n )\n\n return new sherpa.GenerationConfig({\n speed: options.speed,\n numSteps: options.numSteps ?? DEFAULT_NUM_STEPS,\n referenceAudio: voice.samples,\n referenceSampleRate: voice.sampleRate,\n extra: {\n max_reference_audio_len:\n options.referenceLength ?? DEFAULT_REFERENCE_LENGTH,\n ...(options.seed === undefined ? {} : { seed: options.seed }),\n },\n })\n }\n\n private getSynthesizerOptions(): SynthesizerOptions {\n return {\n modelDir: this.options.modelDir,\n numThreads: this.options.numThreads,\n provider: this.options.provider,\n }\n }\n}\n","import { SharedInstances } from '@micdrop/server'\nimport { existsSync } from 'fs'\nimport sherpa, { type OfflineTts, type Wave } from 'sherpa-onnx-node'\nimport { resolveModelFiles } from './modelFiles'\n\nexport interface SynthesizerOptions {\n modelDir: string\n numThreads?: number\n provider?: string\n debug?: boolean\n}\n\nconst DEFAULT_NUM_THREADS = 2\nconst DEFAULT_PROVIDER = 'cpu'\n// Voices already turned into an embedding, kept by the addon between calls\nconst VOICE_CACHE_CAPACITY = 10\n\n/**\n * One instance per configuration, shared by every call.\n *\n * The weights weigh close to two hundred megabytes, so holding a second copy\n * for a second call would cost more memory than the language model it sits\n * next to. Inference is bound to a couple of threads anyway, and two calls\n * generating at once compete for the same cores whether they share the model\n * or not.\n */\nconst synthesizers = new SharedInstances<OfflineTts>()\n\nexport function loadSynthesizer(\n options: SynthesizerOptions\n): Promise<OfflineTts> {\n return synthesizers.load(\n [\n options.modelDir,\n options.numThreads ?? null,\n options.provider ?? null,\n options.debug ?? null,\n ],\n // createAsync loads the graphs on a worker thread, so setting a call up\n // does not hold the event loop for the half second it takes\n () =>\n sherpa.OfflineTts.createAsync({\n model: {\n pocket: {\n ...resolveModelFiles(options.modelDir),\n voiceEmbeddingCacheCapacity: VOICE_CACHE_CAPACITY,\n },\n numThreads: options.numThreads ?? DEFAULT_NUM_THREADS,\n provider: options.provider ?? DEFAULT_PROVIDER,\n debug: options.debug ?? false,\n },\n // Sentences are already split upstream, one per synthesis\n maxNumSentences: 1,\n })\n )\n}\n\n/**\n * Reference voices read from disk, kept by path.\n *\n * The samples are handed to the addon on every sentence, which is where the\n * voice embedding gets cached, so what is saved here is only the file read.\n */\nconst voices = new Map<string, Wave>()\n\nexport function loadVoice(path: string): Wave {\n const existing = voices.get(path)\n if (existing) return existing\n\n if (!existsSync(path)) {\n throw new Error(`Pocket TTS reference voice not found: \"${path}\"`)\n }\n\n const wave = sherpa.readWave(path)\n voices.set(path, wave)\n return wave\n}\n"],"mappings":";AAAA,SAAS,kBAAkB;AAC3B,SAAS,YAAY;AAarB,IAAM,UAAU;AAAA,EACd,QAAQ;AAAA,EACR,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,SAAS;AAAA,EACT,iBAAiB;AACnB;AAEA,IAAM,aAAa;AAAA,EACjB,WAAW;AAAA,EACX,iBAAiB;AACnB;AAEO,SAAS,kBACd,UACwD;AACxD,QAAM,QAAgC,CAAC;AACvC,QAAM,UAAoB,CAAC;AAE3B,aAAW,CAAC,KAAK,IAAI,KAAK,OAAO,QAAQ,OAAO,GAAG;AACjD,UAAM,QAAQ,CAAC,GAAG,IAAI,cAAc,GAAG,IAAI,OAAO,EAC/C,IAAI,CAAC,SAAS,KAAK,UAAU,IAAI,CAAC,EAClC,KAAK,UAAU;AAClB,QAAI,MAAO,OAAM,GAAG,IAAI;AAAA,QACnB,SAAQ,KAAK,GAAG,IAAI,iBAAiB,IAAI,OAAO;AAAA,EACvD;AAEA,aAAW,CAAC,KAAK,IAAI,KAAK,OAAO,QAAQ,UAAU,GAAG;AACpD,UAAM,OAAO,KAAK,UAAU,IAAI;AAChC,QAAI,WAAW,IAAI,EAAG,OAAM,GAAG,IAAI;AAAA,QAC9B,SAAQ,KAAK,IAAI;AAAA,EACxB;AAEA,MAAI,QAAQ,SAAS,GAAG;AACtB,UAAM,IAAI;AAAA,MACR,wCAAwC,QAAQ,MAAM,QAAQ,KAAK,IAAI,CAAC;AAAA,IAE1E;AAAA,EACF;AAEA,SAAO;AACT;AAUO,IAAM,iBAAiB;AAAA,EAC5B,MAAM;AAAA,EACN,OAAO;AAAA,EACP,cAAc;AAChB;AAKO,SAAS,iBACd,UACA,OACQ;AACR,QAAM,UAAU,eAAe,KAA2B;AAC1D,SAAO,UAAU,KAAK,UAAU,OAAO,IAAI;AAC7C;;;AChFA,SAAS,gBAAgB,gBAAgB,mBAAmB;AAC5D,OAAOA,aAAuC;;;ACD9C,SAAS,uBAAuB;AAChC,SAAS,cAAAC,mBAAkB;AAC3B,OAAO,YAA4C;AAUnD,IAAM,sBAAsB;AAC5B,IAAM,mBAAmB;AAEzB,IAAM,uBAAuB;AAW7B,IAAM,eAAe,IAAI,gBAA4B;AAE9C,SAAS,gBACd,SACqB;AACrB,SAAO,aAAa;AAAA,IAClB;AAAA,MACE,QAAQ;AAAA,MACR,QAAQ,cAAc;AAAA,MACtB,QAAQ,YAAY;AAAA,MACpB,QAAQ,SAAS;AAAA,IACnB;AAAA;AAAA;AAAA,IAGA,MACE,OAAO,WAAW,YAAY;AAAA,MAC5B,OAAO;AAAA,QACL,QAAQ;AAAA,UACN,GAAG,kBAAkB,QAAQ,QAAQ;AAAA,UACrC,6BAA6B;AAAA,QAC/B;AAAA,QACA,YAAY,QAAQ,cAAc;AAAA,QAClC,UAAU,QAAQ,YAAY;AAAA,QAC9B,OAAO,QAAQ,SAAS;AAAA,MAC1B;AAAA;AAAA,MAEA,iBAAiB;AAAA,IACnB,CAAC;AAAA,EACL;AACF;AAQA,IAAM,SAAS,oBAAI,IAAkB;AAE9B,SAAS,UAAU,MAAoB;AAC5C,QAAM,WAAW,OAAO,IAAI,IAAI;AAChC,MAAI,SAAU,QAAO;AAErB,MAAI,CAACC,YAAW,IAAI,GAAG;AACrB,UAAM,IAAI,MAAM,0CAA0C,IAAI,GAAG;AAAA,EACnE;AAEA,QAAM,OAAO,OAAO,SAAS,IAAI;AACjC,SAAO,IAAI,MAAM,IAAI;AACrB,SAAO;AACT;;;ADlBA,IAAM,gBAAoC;AAC1C,IAAM,oBAAoB;AAC1B,IAAM,2BAA2B;AACjC,IAAM,qBAAqB;AAEpB,IAAM,YAAN,cAAwB,YAAY;AAAA,EACzC,YAA6B,SAA2B;AACtD,UAAM;AADqB;AAI3B,sBAAkB,QAAQ,QAAQ;AAElC,oBAAgB,KAAK,sBAAsB,CAAC,EAAE,MAAM,CAAC,UAAU;AAC7D,cAAQ,MAAM,qCAAqC,KAAK;AAAA,IAC1D,CAAC;AAED,QAAI,QAAQ,WAAW,OAAO;AAC5B,WAAK,OAAO,EACT,KAAK,MAAM,KAAK,IAAI,iBAAiB,CAAC,EACtC,MAAM,MAAM;AAAA,MAAC,CAAC;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,MAAgB,WACd,MACA,QACoB;AACpB,UAAM,cAAc,MAAM,gBAAgB,KAAK,sBAAsB,CAAC;AAEtE,QAAI,OAAO,QAAS;AAIpB,UAAM,YAAY,IAAI;AAAA,MACpB,YAAY;AAAA,MACZ;AAAA,IACF;AAEA,UAAM,YAAY,cAAc;AAAA,MAC9B;AAAA;AAAA;AAAA,MAGA,sBAAsB;AAAA,MACtB,kBAAkB,KAAK,oBAAoB;AAAA,MAC3C,YAAY,CAAC,EAAE,QAAQ,MAAM;AAC3B,cAAM,QAAQ,UAAU,QAAQ,eAAe,OAAO,CAAC;AAEvD,eAAO,KAAK,UAAU,KAAK,IAAI,IAAI;AAAA,MACrC;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAc,SAAwB;AACpC,UAAM,cAAc,MAAM,gBAAgB,KAAK,sBAAsB,CAAC;AACtE,UAAM,YAAY,cAAc;AAAA,MAC9B,MAAM;AAAA,MACN,kBAAkB,KAAK,oBAAoB;AAAA,IAC7C,CAAC;AAAA,EACH;AAAA,EAEQ,sBAAwC;AAC9C,UAAM,EAAE,QAAQ,IAAI;AACpB,UAAM,QAAQ;AAAA,MACZ,iBAAiB,QAAQ,UAAU,QAAQ,SAAS,aAAa;AAAA,IACnE;AAEA,WAAO,IAAIC,QAAO,iBAAiB;AAAA,MACjC,OAAO,QAAQ;AAAA,MACf,UAAU,QAAQ,YAAY;AAAA,MAC9B,gBAAgB,MAAM;AAAA,MACtB,qBAAqB,MAAM;AAAA,MAC3B,OAAO;AAAA,QACL,yBACE,QAAQ,mBAAmB;AAAA,QAC7B,GAAI,QAAQ,SAAS,SAAY,CAAC,IAAI,EAAE,MAAM,QAAQ,KAAK;AAAA,MAC7D;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,wBAA4C;AAClD,WAAO;AAAA,MACL,UAAU,KAAK,QAAQ;AAAA,MACvB,YAAY,KAAK,QAAQ;AAAA,MACzB,UAAU,KAAK,QAAQ;AAAA,IACzB;AAAA,EACF;AACF;","names":["sherpa","existsSync","existsSync","sherpa"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@micdrop/pocket-tts",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "Local Pocket TTS text to speech for @micdrop/server",
|
|
5
5
|
"author": "Godefroy de Compreignac",
|
|
6
6
|
"license": "MIT",
|
|
@@ -40,7 +40,7 @@
|
|
|
40
40
|
"sherpa-onnx-node": "^1.13.6"
|
|
41
41
|
},
|
|
42
42
|
"peerDependencies": {
|
|
43
|
-
"@micdrop/server": ">=2.
|
|
43
|
+
"@micdrop/server": ">=2.7.0"
|
|
44
44
|
},
|
|
45
45
|
"devDependencies": {
|
|
46
46
|
"@types/node": "^22.19.13",
|