@senseihq/emaki-cli 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/tts-io.js ADDED
@@ -0,0 +1,51 @@
1
+ // What every text-to-speech provider shares: the TTSError a check or a synthesis throws, running a command, a take
2
+ // in the shape fit and build expect, keys from the environment, and the HTTP calls the cloud providers make.
3
+ import { spawnSync } from 'node:child_process'
4
+ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
5
+ import { tmpdir } from 'node:os'
6
+ import { join } from 'node:path'
7
+
8
+ export class TTSError extends Error {}
9
+ export const fail = (m) => { throw new TTSError(m) }
10
+ export const lang = (locale) => locale.split('-')[0]
11
+
12
+ export function sh(cmd, args, input) {
13
+ const r = spawnSync(cmd, args, { input, encoding: input == null ? 'utf-8' : undefined, maxBuffer: 256 * 1024 * 1024 })
14
+ if (r.error) fail(`${cmd}: ${r.error.code === 'ENOENT' ? 'not installed' : r.error.message}`)
15
+ if (r.status) fail(`${cmd}: ${String(r.stderr ?? '').slice(-400)}`)
16
+ return r.stdout
17
+ }
18
+
19
+ /** Any audio file or bytes → 48 kHz mono WAV with 0.1 s head air at dst. */
20
+ export function toTake(src, dst) {
21
+ const dir = mkdtempSync(join(tmpdir(), 'emaki-'))
22
+ try {
23
+ let file = src
24
+ if (typeof src !== 'string') { file = join(dir, 'raw'); writeFileSync(file, src) }
25
+ sh('ffmpeg', ['-v', 'error', '-y', '-i', file, '-af', 'adelay=100|100', '-ar', '48000', '-ac', '1', dst])
26
+ } finally {
27
+ rmSync(dir, { recursive: true, force: true })
28
+ }
29
+ }
30
+
31
+ export function inTemp(fn) {
32
+ const dir = mkdtempSync(join(tmpdir(), 'emaki-'))
33
+ try { return fn(dir) } finally { rmSync(dir, { recursive: true, force: true }) }
34
+ }
35
+
36
+ export function env(...names) {
37
+ for (const n of names) if (process.env[n]) return process.env[n]
38
+ return fail(`set ${names.join(' or ')} in the environment (never in files)`)
39
+ }
40
+
41
+ export async function post(url, body, headers, as = 'bytes') {
42
+ const r = await fetch(url, { method: 'POST', body: typeof body === 'string' || body instanceof Uint8Array ? body : JSON.stringify(body), headers })
43
+ if (!r.ok) fail(`${new URL(url).host}: HTTP ${r.status} ${(await r.text()).slice(0, 300)}`)
44
+ return as === 'json' ? r.json() : Buffer.from(await r.arrayBuffer())
45
+ }
46
+
47
+ export async function get(url, headers) {
48
+ const r = await fetch(url, { headers })
49
+ if (!r.ok) fail(`${new URL(url).host}: HTTP ${r.status}`)
50
+ return r.json()
51
+ }
@@ -0,0 +1,85 @@
1
+ // The offline text-to-speech providers: Piper, Kokoro, macOS say and eSpeak NG. Each runs a local command and turns
2
+ // its output into a take (toTake). check(cfg, locale) throws TTSError when not ready; synth(text, dst, cfg, locale).
3
+ import { spawnSync } from 'node:child_process'
4
+ import { existsSync } from 'node:fs'
5
+ import { join } from 'node:path'
6
+ import { fail, inTemp, lang, sh, toTake } from './tts-io.js'
7
+ import { which } from './util.js'
8
+
9
+ const piper = {
10
+ label: 'Piper (offline)',
11
+ check(cfg) {
12
+ if (!which('piper')) fail('install it: pip install piper-tts')
13
+ if (!cfg.model || !existsSync(cfg.model)) fail(`voice model not found: "${cfg.model ?? ''}". Download a voice (.onnx and .onnx.json) from the Piper voices list and set "model"`)
14
+ },
15
+ async synth(text, dst, cfg) {
16
+ inTemp((d) => {
17
+ const raw = join(d, 'raw.wav')
18
+ const args = ['--model', cfg.model, '--output_file', raw]
19
+ if (cfg.speaker != null) args.push('--speaker', String(cfg.speaker))
20
+ if (cfg.length_scale) args.push('--length_scale', String(cfg.length_scale)) // >1 is slower
21
+ sh('piper', args, text)
22
+ if (!existsSync(raw)) fail('piper wrote nothing')
23
+ toTake(raw, dst)
24
+ })
25
+ },
26
+ voices: async () => ['Piper voices are files: pick one for this language from the Piper voice samples page, download its .onnx and .onnx.json, set "model".']
27
+ }
28
+
29
+ const KOKORO_LANG = { en: 'a', 'en-GB': 'b', es: 'e', fr: 'f', hi: 'h', it: 'i', ja: 'j', pt: 'p', zh: 'z' }
30
+ const KOKORO_PY = `import sys, json, numpy as np, soundfile as sf
31
+ from kokoro import KPipeline
32
+ a = json.loads(sys.argv[1]); text = sys.stdin.read()
33
+ pipe = KPipeline(lang_code=a['code'])
34
+ audio = [x for _, _, x in pipe(text, voice=a['voice'], speed=a['speed'])]
35
+ sf.write(a['out'], np.concatenate(audio), 24000)`
36
+ const kokoro = {
37
+ label: 'Kokoro (offline)',
38
+ check(cfg, locale) {
39
+ const py = cfg.python ?? 'python3'
40
+ const r = spawnSync(py, ['-c', 'import kokoro, soundfile'], { encoding: 'utf-8' })
41
+ if (r.error || r.status) fail(`install it: ${py} -m pip install kokoro soundfile (and espeak-ng for some languages)`)
42
+ if (!KOKORO_LANG[locale] && !KOKORO_LANG[lang(locale)]) fail(`kokoro has no voices for ${locale}`)
43
+ },
44
+ async synth(text, dst, cfg, locale) {
45
+ inTemp((d) => {
46
+ const out = join(d, 'raw.wav')
47
+ const code = cfg.lang_code ?? KOKORO_LANG[locale] ?? KOKORO_LANG[lang(locale)]
48
+ sh(cfg.python ?? 'python3', ['-c', KOKORO_PY, JSON.stringify({ code, voice: cfg.voice ?? 'af_heart', speed: cfg.speed ?? 1, out })], text)
49
+ toTake(out, dst)
50
+ })
51
+ },
52
+ voices: async () => ['af_heart, af_bella, am_michael, bf_emma, bm_george (en); ef_dora, em_alex (es); ff_siwis (fr); if_sara (it); pf_dora (pt); jf_alpha (ja); zf_xiaobei (zh); see the Kokoro VOICES list']
53
+ }
54
+
55
+ const macos = {
56
+ label: 'macOS say (offline, drafts)',
57
+ check() { if (!which('say')) fail('`say` exists only on macOS') },
58
+ async synth(text, dst, cfg) {
59
+ inTemp((d) => {
60
+ const raw = join(d, 'raw.aiff')
61
+ sh('say', ['-o', raw, ...(cfg.voice ? ['-v', cfg.voice] : []), ...(cfg.rate ? ['-r', String(cfg.rate)] : []), text])
62
+ toTake(raw, dst)
63
+ })
64
+ },
65
+ async voices(cfg, locale) {
66
+ const all = String(sh('say', ['-v', '?'])).split('\n').filter(Boolean)
67
+ const l = lang(locale), mine = all.filter((x) => x.includes(` ${l}_`) || x.includes(` ${l}-`))
68
+ return mine.length ? mine : all.slice(0, 40)
69
+ }
70
+ }
71
+
72
+ const espeak = {
73
+ label: 'eSpeak NG (offline, robotic)',
74
+ check() { if (!which('espeak-ng')) fail('install espeak-ng (brew/apt/choco)') },
75
+ async synth(text, dst, cfg, locale) {
76
+ inTemp((d) => {
77
+ const raw = join(d, 'raw.wav')
78
+ sh('espeak-ng', ['-v', cfg.voice ?? locale, '-s', String(cfg.speed ?? 160), '-w', raw, text])
79
+ toTake(raw, dst)
80
+ })
81
+ },
82
+ voices: async (cfg, locale) => String(sh('espeak-ng', [`--voices=${lang(locale)}`])).split('\n').filter(Boolean)
83
+ }
84
+
85
+ export { piper, kokoro, macos, espeak }
package/src/tts.js CHANGED
@@ -20,246 +20,14 @@
20
20
  // | google | cloud (Google) | GOOGLE_TTS_API_KEY, or `gcloud auth login` |
21
21
  // | polly | cloud (AWS) | the `aws` CLI with credentials |
22
22
  // | manual | a person | recorded take-NN.wav files |
23
- import { spawnSync } from 'node:child_process'
24
- import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
25
- import { tmpdir } from 'node:os'
26
- import { join } from 'node:path'
27
- import { which } from './util.js'
23
+ import { azure, elevenlabs, google, openai, polly } from './tts-cloud.js'
24
+ import { fail, lang, TTSError, toTake } from './tts-io.js'
25
+ import { espeak, kokoro, macos, piper } from './tts-offline.js'
28
26
 
29
- export class TTSError extends Error {}
30
- const fail = (m) => { throw new TTSError(m) }
31
- const lang = (locale) => locale.split('-')[0]
32
-
33
- function sh(cmd, args, input) {
34
- const r = spawnSync(cmd, args, { input, encoding: input == null ? 'utf-8' : undefined, maxBuffer: 256 * 1024 * 1024 })
35
- if (r.error) fail(`${cmd}: ${r.error.code === 'ENOENT' ? 'not installed' : r.error.message}`)
36
- if (r.status) fail(`${cmd}: ${String(r.stderr ?? '').slice(-400)}`)
37
- return r.stdout
38
- }
39
-
40
- /** Any audio file or bytes → 48 kHz mono WAV with 0.1 s head air at dst. */
41
- export function toTake(src, dst) {
42
- const dir = mkdtempSync(join(tmpdir(), 'emaki-'))
43
- try {
44
- let file = src
45
- if (typeof src !== 'string') { file = join(dir, 'raw'); writeFileSync(file, src) }
46
- sh('ffmpeg', ['-v', 'error', '-y', '-i', file, '-af', 'adelay=100|100', '-ar', '48000', '-ac', '1', dst])
47
- } finally {
48
- rmSync(dir, { recursive: true, force: true })
49
- }
50
- }
51
- function inTemp(fn) {
52
- const dir = mkdtempSync(join(tmpdir(), 'emaki-'))
53
- try { return fn(dir) } finally { rmSync(dir, { recursive: true, force: true }) }
54
- }
55
-
56
- function env(...names) {
57
- for (const n of names) if (process.env[n]) return process.env[n]
58
- return fail(`set ${names.join(' or ')} in the environment (never in files)`)
59
- }
60
-
61
- async function post(url, body, headers, as = 'bytes') {
62
- const r = await fetch(url, { method: 'POST', body: typeof body === 'string' || body instanceof Uint8Array ? body : JSON.stringify(body), headers })
63
- if (!r.ok) fail(`${new URL(url).host}: HTTP ${r.status} ${(await r.text()).slice(0, 300)}`)
64
- return as === 'json' ? r.json() : Buffer.from(await r.arrayBuffer())
65
- }
66
- async function get(url, headers) {
67
- const r = await fetch(url, { headers })
68
- if (!r.ok) fail(`${new URL(url).host}: HTTP ${r.status}`)
69
- return r.json()
70
- }
27
+ export { TTSError, toTake }
71
28
 
72
29
  // ── Providers: check(cfg, locale) throws TTSError when not ready; synth(text, dst, cfg, locale, ctx) ─────────────
73
-
74
- const piper = {
75
- label: 'Piper (offline)',
76
- check(cfg) {
77
- if (!which('piper')) fail('install it: pip install piper-tts')
78
- if (!cfg.model || !existsSync(cfg.model)) fail(`voice model not found: "${cfg.model ?? ''}". Download a voice (.onnx and .onnx.json) from the Piper voices list and set "model"`)
79
- },
80
- async synth(text, dst, cfg) {
81
- inTemp((d) => {
82
- const raw = join(d, 'raw.wav')
83
- const args = ['--model', cfg.model, '--output_file', raw]
84
- if (cfg.speaker != null) args.push('--speaker', String(cfg.speaker))
85
- if (cfg.length_scale) args.push('--length_scale', String(cfg.length_scale)) // >1 is slower
86
- sh('piper', args, text)
87
- if (!existsSync(raw)) fail('piper wrote nothing')
88
- toTake(raw, dst)
89
- })
90
- },
91
- voices: async () => ['Piper voices are files: pick one for this language from the Piper voice samples page, download its .onnx and .onnx.json, set "model".']
92
- }
93
-
94
- const KOKORO_LANG = { en: 'a', 'en-GB': 'b', es: 'e', fr: 'f', hi: 'h', it: 'i', ja: 'j', pt: 'p', zh: 'z' }
95
- const KOKORO_PY = `import sys, json, numpy as np, soundfile as sf
96
- from kokoro import KPipeline
97
- a = json.loads(sys.argv[1]); text = sys.stdin.read()
98
- pipe = KPipeline(lang_code=a['code'])
99
- audio = [x for _, _, x in pipe(text, voice=a['voice'], speed=a['speed'])]
100
- sf.write(a['out'], np.concatenate(audio), 24000)`
101
- const kokoro = {
102
- label: 'Kokoro (offline)',
103
- check(cfg, locale) {
104
- const py = cfg.python ?? 'python3'
105
- const r = spawnSync(py, ['-c', 'import kokoro, soundfile'], { encoding: 'utf-8' })
106
- if (r.error || r.status) fail(`install it: ${py} -m pip install kokoro soundfile (and espeak-ng for some languages)`)
107
- if (!KOKORO_LANG[locale] && !KOKORO_LANG[lang(locale)]) fail(`kokoro has no voices for ${locale}`)
108
- },
109
- async synth(text, dst, cfg, locale) {
110
- inTemp((d) => {
111
- const out = join(d, 'raw.wav')
112
- const code = cfg.lang_code ?? KOKORO_LANG[locale] ?? KOKORO_LANG[lang(locale)]
113
- sh(cfg.python ?? 'python3', ['-c', KOKORO_PY, JSON.stringify({ code, voice: cfg.voice ?? 'af_heart', speed: cfg.speed ?? 1, out })], text)
114
- toTake(out, dst)
115
- })
116
- },
117
- voices: async () => ['af_heart, af_bella, am_michael, bf_emma, bm_george (en); ef_dora, em_alex (es); ff_siwis (fr); if_sara (it); pf_dora (pt); jf_alpha (ja); zf_xiaobei (zh); see the Kokoro VOICES list']
118
- }
119
-
120
- const macos = {
121
- label: 'macOS say (offline, drafts)',
122
- check() { if (!which('say')) fail('`say` exists only on macOS') },
123
- async synth(text, dst, cfg) {
124
- inTemp((d) => {
125
- const raw = join(d, 'raw.aiff')
126
- sh('say', ['-o', raw, ...(cfg.voice ? ['-v', cfg.voice] : []), ...(cfg.rate ? ['-r', String(cfg.rate)] : []), text])
127
- toTake(raw, dst)
128
- })
129
- },
130
- async voices(cfg, locale) {
131
- const all = String(sh('say', ['-v', '?'])).split('\n').filter(Boolean)
132
- const l = lang(locale), mine = all.filter((x) => x.includes(` ${l}_`) || x.includes(` ${l}-`))
133
- return mine.length ? mine : all.slice(0, 40)
134
- }
135
- }
136
-
137
- const espeak = {
138
- label: 'eSpeak NG (offline, robotic)',
139
- check() { if (!which('espeak-ng')) fail('install espeak-ng (brew/apt/choco)') },
140
- async synth(text, dst, cfg, locale) {
141
- inTemp((d) => {
142
- const raw = join(d, 'raw.wav')
143
- sh('espeak-ng', ['-v', cfg.voice ?? locale, '-s', String(cfg.speed ?? 160), '-w', raw, text])
144
- toTake(raw, dst)
145
- })
146
- },
147
- voices: async (cfg, locale) => String(sh('espeak-ng', [`--voices=${lang(locale)}`])).split('\n').filter(Boolean)
148
- }
149
-
150
- const elevenlabs = {
151
- label: 'ElevenLabs',
152
- check(cfg) {
153
- env('ELEVENLABS_API_KEY')
154
- if (!cfg.voice_id && !cfg.voice) fail('set "voice_id"')
155
- },
156
- async synth(text, dst, cfg, locale, ctx) {
157
- const body = {
158
- text, model_id: cfg.model ?? 'eleven_multilingual_v2', language_code: lang(locale),
159
- voice_settings: { stability: cfg.stability ?? 0.55, similarity_boost: cfg.similarity ?? 0.8 }
160
- }
161
- if (ctx.prev) body.previous_text = ctx.prev
162
- if (ctx.next) body.next_text = ctx.next
163
- const audio = await post(`https://api.elevenlabs.io/v1/text-to-speech/${cfg.voice_id ?? cfg.voice}?output_format=mp3_44100_128`, body,
164
- { 'xi-api-key': env('ELEVENLABS_API_KEY'), 'content-type': 'application/json' })
165
- toTake(audio, dst)
166
- },
167
- async voices() {
168
- const data = await get('https://api.elevenlabs.io/v1/voices', { 'xi-api-key': env('ELEVENLABS_API_KEY') })
169
- return (data.voices ?? []).map((v) => `${v.voice_id} ${v.name} ${JSON.stringify(v.labels ?? {})}`)
170
- }
171
- }
172
-
173
- const openai = {
174
- label: 'OpenAI (or Azure OpenAI)',
175
- check() {
176
- if (process.env.AZURE_OPENAI_ENDPOINT) env('AZURE_OPENAI_API_KEY')
177
- else env('OPENAI_API_KEY')
178
- },
179
- async synth(text, dst, cfg) {
180
- const body = { model: cfg.model ?? 'gpt-4o-mini-tts', voice: cfg.voice ?? 'alloy', input: text, response_format: 'wav' }
181
- if (cfg.instructions) body.instructions = cfg.instructions // e.g. "Calm, warm, unhurried explainer."
182
- let url, headers
183
- if (process.env.AZURE_OPENAI_ENDPOINT) {
184
- url = `${process.env.AZURE_OPENAI_ENDPOINT.replace(/\/$/, '')}/openai/deployments/${cfg.deployment ?? body.model}/audio/speech?api-version=${cfg.api_version ?? '2025-03-01-preview'}`
185
- headers = { 'api-key': env('AZURE_OPENAI_API_KEY'), 'content-type': 'application/json' }
186
- } else {
187
- url = 'https://api.openai.com/v1/audio/speech'
188
- headers = { authorization: `Bearer ${env('OPENAI_API_KEY')}`, 'content-type': 'application/json' }
189
- }
190
- toTake(await post(url, body, headers), dst)
191
- },
192
- voices: async () => ['alloy, ash, ballad, coral, echo, fable, nova, onyx, sage, shimmer, verse (all speak every supported language; set "instructions" for tone)']
193
- }
194
-
195
- const xml = (s) => s.replace(/[<>&'"]/g, (c) => ({ '<': '&lt;', '>': '&gt;', '&': '&amp;', "'": '&apos;', '"': '&quot;' })[c])
196
- const azure = {
197
- label: 'Azure AI Speech',
198
- check(cfg) {
199
- env('AZURE_SPEECH_KEY'); env('AZURE_SPEECH_REGION')
200
- if (!cfg.voice) fail('set "voice", e.g. "en-US-AvaMultilingualNeural" or "es-ES-ElviraNeural"')
201
- },
202
- async synth(text, dst, cfg, locale) {
203
- const ssml = `<speak version='1.0' xml:lang='${locale}'><voice name='${cfg.voice}'><prosody rate='${cfg.rate ?? '0%'}'>${xml(text)}</prosody></voice></speak>`
204
- const audio = await post(`https://${env('AZURE_SPEECH_REGION')}.tts.speech.microsoft.com/cognitiveservices/v1`, ssml, {
205
- 'Ocp-Apim-Subscription-Key': env('AZURE_SPEECH_KEY'), 'content-type': 'application/ssml+xml',
206
- 'X-Microsoft-OutputFormat': 'riff-48khz-16bit-mono-pcm', 'user-agent': 'emaki'
207
- })
208
- toTake(audio, dst)
209
- },
210
- async voices(cfg, locale) {
211
- const data = await get(`https://${env('AZURE_SPEECH_REGION')}.tts.speech.microsoft.com/cognitiveservices/voices/list`, { 'Ocp-Apim-Subscription-Key': env('AZURE_SPEECH_KEY') })
212
- return data.filter((v) => v.Locale.startsWith(locale) || v.Locale.split('-')[0] === lang(locale)).map((v) => `${v.ShortName} ${v.Gender}`)
213
- }
214
- }
215
-
216
- function googleAuth() {
217
- if (process.env.GOOGLE_TTS_API_KEY) return { 'X-Goog-Api-Key': process.env.GOOGLE_TTS_API_KEY }
218
- if (which('gcloud')) {
219
- const tok = spawnSync('gcloud', ['auth', 'print-access-token'], { encoding: 'utf-8' }).stdout?.trim()
220
- if (tok) return { authorization: `Bearer ${tok}` }
221
- }
222
- return fail('set GOOGLE_TTS_API_KEY, or sign in with `gcloud auth login`')
223
- }
224
- const google = {
225
- label: 'Google Cloud Text-to-Speech',
226
- check(cfg) {
227
- googleAuth()
228
- if (!cfg.voice) fail('set "voice", e.g. "en-US-Neural2-F" or "es-ES-Chirp3-HD-Aoede"')
229
- },
230
- async synth(text, dst, cfg) {
231
- const body = {
232
- input: { text }, voice: { languageCode: cfg.voice.split('-').slice(0, 2).join('-'), name: cfg.voice },
233
- audioConfig: { audioEncoding: 'LINEAR16', sampleRateHertz: 48000, speakingRate: cfg.speed ?? 1 }
234
- }
235
- const data = await post('https://texttospeech.googleapis.com/v1/text:synthesize', body, { ...googleAuth(), 'content-type': 'application/json' }, 'json')
236
- toTake(Buffer.from(data.audioContent, 'base64'), dst)
237
- },
238
- async voices(cfg, locale) {
239
- const data = await get(`https://texttospeech.googleapis.com/v1/voices?languageCode=${locale}`, googleAuth())
240
- return (data.voices ?? []).map((v) => `${v.name} ${v.ssmlGender}`)
241
- }
242
- }
243
-
244
- const polly = {
245
- label: 'Amazon Polly',
246
- check(cfg) {
247
- if (!which('aws')) fail('install the AWS CLI and sign in (aws configure / SSO); credentials stay in your AWS profile')
248
- if (!cfg.voice) fail('set "voice", e.g. "Joanna" (en-US) or "Lucia" (es-ES)')
249
- },
250
- async synth(text, dst, cfg, locale) {
251
- inTemp((d) => {
252
- const raw = join(d, 'raw.mp3')
253
- sh('aws', ['polly', 'synthesize-speech', '--output-format', 'mp3', '--voice-id', cfg.voice, '--engine', cfg.engine ?? 'neural',
254
- '--language-code', cfg.language ?? locale, '--text', text, ...(cfg.region ? ['--region', cfg.region] : []), raw])
255
- toTake(raw, dst)
256
- })
257
- },
258
- async voices(cfg, locale) {
259
- const out = JSON.parse(String(sh('aws', ['polly', 'describe-voices', '--language-code', cfg.language ?? locale, '--engine', cfg.engine ?? 'neural', ...(cfg.region ? ['--region', cfg.region] : [])])))
260
- return out.Voices.map((v) => `${v.Id} ${v.Gender}`)
261
- }
262
- }
30
+ // The offline ones are in tts-offline.js, the cloud ones in tts-cloud.js; manual needs nothing.
263
31
 
264
32
  const manual = {
265
33
  label: 'Recorded by a person',
package/src/voice.js CHANGED
@@ -1,14 +1,14 @@
1
- // The narration: choose a voice per locale, synthesise takes, fit scene times to them, and build the final videos
2
- // (voice + score + captions, -14 LUFS).
3
- import { copyFileSync, existsSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs'
4
- import { join, resolve } from 'node:path'
5
- import { SUFFIX } from './config.js'
6
- import {
7
- buildArgs, buildPlan, chaptersText, description, ffmeta, planTable, posterSteps, storyLink, webArgs, webNote, webVoiceArgs, ytChapters
8
- } from './media.js'
1
+ // The narration: choose a voice per locale and synthesise the takes. Fitting scene times to them (fit.js) and
2
+ // building the final videos (build-video.js: voice + score + captions, -14 LUFS) are re-exported from here.
3
+ import { existsSync, mkdirSync, writeFileSync } from 'node:fs'
4
+ import { join } from 'node:path'
9
5
  import { OFFLINE, provider, TTSError, voiceConfig } from './tts.js'
10
- import { localizeScene, motionEnd } from '@senseihq/emaki'
11
- import { die, hasAudio, load, mediaLength, pad2, run, save } from './util.js'
6
+ import { die, load, mediaLength, pad2, save } from './util.js'
7
+
8
+ // The steps after the takes, re-exported: the timing (fit.js), the final videos (build-video.js), the takes (takes.js).
9
+ export { fit, voiceWeb } from './fit.js'
10
+ export { build, captions } from './build-video.js'
11
+ export { takesFor } from './takes.js'
12
12
 
13
13
  export const NO_PROVIDER = (loc) => `no voice provider is set for ${loc}. Choose one and record it with set-voice, for example:
14
14
  offline piper (good, free) kokoro (very natural, free) macos (Mac drafts) espeak (robotic drafts)
@@ -104,192 +104,3 @@ export async function voice(p, slug, { locale, only, provider: override, opts =
104
104
  writeFileSync(join(takes, 'provider.json'), JSON.stringify(keep, null, 1) + '\n')
105
105
  return made
106
106
  }
107
-
108
- /** scene id → [take file, voice line] for the takes that exist. */
109
- export function takesFor(p, slug, locale) {
110
- const d = p.filmDir(slug), lines = load(join(d, 'voice', `${locale}.json`)), folder = p.prod(slug, locale, 'takes')
111
- const out = new Map()
112
- lines.forEach((l, i) => {
113
- for (const ext of ['wav', 'mp3', 'm4a', 'aiff']) {
114
- const f = join(folder, `take-${pad2(i + 1)}.${ext}`)
115
- if (existsSync(f)) { out.set(l.scene, [f, l]); break }
116
- }
117
- })
118
- return out
119
- }
120
-
121
- /** The narration for the browser: each take as voice/<locale>/<scene>.m4a beside the film, and in timing.<locale>.json
122
- * each narrated scene's `voice` start (the same lead the videos use), so the story player and article figures speak
123
- * in step with the picture. A scene whose take is gone loses its file and its start. Run after `fit`. */
124
- export function voiceWeb(p, slug, { locale, encode = (src, dst) => run('ffmpeg', webVoiceArgs(src, dst)), log = () => {} } = {}) {
125
- const d = p.filmDir(slug), takes = takesFor(p, slug, locale)
126
- if (!takes.size) die(`${slug} has no takes for ${locale}: record or synthesise them first (emaki voice ${slug} --locale ${locale})`)
127
- const film = load(join(d, 'film.json')), dir = join(d, 'voice', locale), tp = join(d, `timing.${locale}.json`)
128
- const timing = existsSync(tp) ? load(tp) : { scenes: {} }
129
- const tsc = (timing.scenes ??= {})
130
- mkdirSync(dir, { recursive: true })
131
- const wrote = [], removed = []
132
- for (const s of film.scenes) {
133
- const dst = join(dir, `${s.id}.m4a`), c = tsc[s.id] ?? {}
134
- if (takes.has(s.id)) {
135
- encode(takes.get(s.id)[0], dst)
136
- wrote.push(p.rel(dst))
137
- log(`${p.rel(dst)}`)
138
- c.voice = p.lead
139
- } else {
140
- if (existsSync(dst)) { rmSync(dst); removed.push(p.rel(dst)) }
141
- delete c.voice
142
- }
143
- if (Object.keys(c).length) tsc[s.id] = c
144
- else delete tsc[s.id]
145
- }
146
- if (!readdirSync(dir).length) rmSync(dir, { recursive: true })
147
- save(tp, timing)
148
- return { wrote, removed, timing: p.rel(tp) }
149
- }
150
-
151
- const up = (v) => Math.ceil(v * 10 - 1e-9) / 10
152
-
153
- /** When scene `s` stops moving, in scene seconds: film.json "settle", else the end of its JSON scene's tracks in this
154
- * locale (word counts follow the translation), else null (a JS scene: its motion cannot be read). */
155
- function settleOf(d, s, locale) {
156
- if (s.settle != null) return s.settle
157
- const f = join(d, `scene-${s.id}.json`)
158
- if (!existsSync(f)) return null
159
- const sp = join(d, `strings/${locale}.json`)
160
- return motionEnd(localizeScene(load(f), existsSync(sp) ? load(sp) : {}))
161
- }
162
-
163
- /** timing.<locale>.json: a narrated scene runs lead + take + tail, longer or shorter than its design, but never ends
164
- * before its motion settles; a scene without a take, or a JS scene without "settle", keeps its design time. */
165
- export function fit(p, slug, { locale, check = false, length = mediaLength } = {}) {
166
- const d = p.filmDir(slug), film = load(join(d, 'film.json')), takes = takesFor(p, slug, locale)
167
- const tp = join(d, `timing.${locale}.json`)
168
- const timing = existsSync(tp) ? load(tp) : { scenes: {} }
169
- const tsc = (timing.scenes ??= {})
170
- const rows = []
171
- for (const s of film.scenes) {
172
- const c = tsc[s.id] ?? {}
173
- const holds = (c.holds ?? []).reduce((n, h) => n + h[1], 0)
174
- const L = takes.has(s.id) ? length(takes.get(s.id)[0]) : 0
175
- const base = s.dur + holds
176
- let runs = base
177
- if (L) {
178
- const settle = settleOf(d, s, locale)
179
- const floor = settle == null ? base : Math.min(base, up(settle + holds))
180
- runs = Math.max(floor, up(p.lead + L + p.tail))
181
- }
182
- if (Math.abs(runs - base) > 1e-6) c.dur = runs
183
- else delete c.dur
184
- if (Object.keys(c).length) tsc[s.id] = c
185
- else delete tsc[s.id]
186
- rows.push({ scene: s.id, design: s.dur, holds, take: L, runs, longer: runs > base + 1e-6, shorter: runs < base - 1e-6 })
187
- }
188
- if (!check) save(tp, timing)
189
- return { rows, total: rows.reduce((n, r) => n + r.runs, 0), path: p.rel(tp), written: !check }
190
- }
191
-
192
- const stamp = (t, sep = '.') => {
193
- const h = Math.floor(t / 3600), m = Math.floor((t % 3600) / 60), s = t % 60
194
- return `${pad2(h)}:${pad2(m)}:${s.toFixed(3).padStart(6, '0')}`.replace('.', sep)
195
- }
196
-
197
- /** Captions from the voice lines, spread over each take by sentence length. */
198
- export function captions(takes, starts, lead, length = mediaLength) {
199
- const vtt = ['WEBVTT', ''], srt = []
200
- let k = 0
201
- for (const [sid, [f, line]] of takes) {
202
- let t0 = starts.get(sid)[0] + lead
203
- const span = length(f) - 0.1
204
- const parts = line.text.split(/(?<=[.!?¿¡;:])\s+/).map((x) => x.trim()).filter(Boolean)
205
- const total = parts.reduce((n, x) => n + x.length, 0)
206
- for (const x of parts) {
207
- const dd = (span * x.length) / total
208
- k++
209
- vtt.push(`${stamp(t0)} --> ${stamp(t0 + dd)}`, x, '')
210
- srt.push(String(k), `${stamp(t0, ',')} --> ${stamp(t0 + dd, ',')}`, x, '')
211
- t0 += dd
212
- }
213
- }
214
- return { vtt: vtt.join('\n'), srt: srt.join('\n') }
215
- }
216
-
217
- /** libass lays SRT out on a 384x288 canvas scaled to the video, so these are in 288ths of the height: landscape 16 ≈
218
- * 60 px text 127 px up; portrait 9 ≈ 60 px text 333 px up (clear of the platform's buttons). */
219
- const burnIn = (srtPath, layout) => {
220
- const size = layout === 'landscape' ? 16 : 9, margin = layout === 'landscape' ? 34 : 50
221
- return `subtitles=${srtPath.replace(/([:\\'])/g, '\\$1')}:force_style='FontName=Inter,FontSize=${size},PrimaryColour=&H00FFFFFF,OutlineColour=&H66000000,BackColour=&H66000000,BorderStyle=3,Outline=5,Shadow=0,MarginV=${margin}'`
222
- }
223
-
224
- /** Final videos per orientation (build.py): the rendered picture with the voice laid `lead` s into each scene, the
225
- * render's music ducked under it, -14 LUFS, captions burned in (and shipped as .srt/.vtt). A take that outruns its
226
- * scene freezes the scene's last frame until it has finished (buildPlan). `check` prints the plan and builds nothing;
227
- * `out` is the output folder (default production/<slug>/<locale>/out/). */
228
- export function build(p, slug, { locale, noCaptions = false, check = false, out: target = undefined, log = console.info, length = mediaLength } = {}) {
229
- const d = p.filmDir(slug), film = load(join(d, 'film.json'))
230
- const tp = join(d, `timing.${locale}.json`)
231
- const tm = existsSync(tp) ? (load(tp).scenes ?? {}) : {}
232
- const takes = takesFor(p, slug, locale)
233
- const plan = buildPlan(film, tm, new Map([...takes].map(([sid, [file]]) => [sid, { file, seconds: length(file) }])), p)
234
- const render = p.prod(slug, locale, 'render')
235
- const pics = ['landscape', 'portrait'].map((layout) => [layout, join(render, `${slug}-${locale}-${SUFFIX[layout]}.mp4`)])
236
- .filter(([, f]) => existsSync(f))
237
- log(`${film.title ?? slug} (${locale}): ${plan.picture.toFixed(1)} s as rendered`)
238
- planTable(plan).forEach((l) => log(l))
239
- if (plan.missing.length) log(`missing takes: ${plan.missing.join(', ')}`)
240
- if (check) {
241
- if (!pics.length) log(`MISSING: no render in ${p.rel(render)}; run \`emaki render ${slug} --locale ${locale}\``)
242
- return { built: [], duration: plan.total, plan }
243
- }
244
- if (!takes.size) die(`no takes in ${p.rel(p.prod(slug, locale, 'takes'))}; run \`emaki voice\` (or record them) first`)
245
- if (!pics.length) die(`no render in ${p.rel(render)}; run \`emaki render ${slug} --locale ${locale}\` first`)
246
- const out = target ? resolve(target) : p.prod(slug, locale, 'out')
247
- mkdirSync(out, { recursive: true })
248
- const voiced = new Map([...takes].filter(([sid]) => plan.rows.some((r) => r.id === sid)))
249
- const starts = new Map(plan.rows.map((r) => [r.id, [r.at, r.tot]]))
250
- const cap = captions(voiced, starts, p.lead, length)
251
- const srtPath = join(out, `${slug}-${locale}.srt`)
252
- writeFileSync(join(out, `${slug}-${locale}.vtt`), cap.vtt)
253
- writeFileSync(srtPath, cap.srt)
254
- const inputs = [], filters = []
255
- ;[...voiced].forEach(([sid, [f]], i) => {
256
- inputs.push('-i', f)
257
- const ms = Math.round((starts.get(sid)[0] + p.lead) * 1000)
258
- filters.push(`[${i}:a]aresample=48000,adelay=${ms}|${ms},apad[v${i}]`)
259
- })
260
- const voicePath = join(out, 'voice.wav')
261
- run('ffmpeg', ['-y', '-v', 'error', ...inputs, '-filter_complex',
262
- filters.join(';') + ';' + [...voiced].map((_, i) => `[v${i}]`).join('') + `amix=inputs=${voiced.size}:normalize=0,atrim=0:${plan.total.toFixed(3)}[vo]`,
263
- '-map', '[vo]', '-ac', '2', voicePath])
264
- const chapters = join(out, 'chapters.ffmeta')
265
- writeFileSync(chapters, ffmeta(plan.rows))
266
- const yt = ytChapters(plan.rows)
267
- writeFileSync(join(out, 'chapters.txt'), chaptersText(yt))
268
- const story = join(p.content, 'stories', locale, `${slug}.json`)
269
- const title = (existsSync(story) ? load(story).title : null) ?? film.title ?? slug
270
- writeFileSync(join(out, 'description.txt'), description({ title, chapters: yt, link: storyLink(p.site, p.cfg, slug, locale) }))
271
- const built = [], videos = {}
272
- for (const [layout, pic] of pics) {
273
- const dst = join(out, `${slug}-${locale}-${SUFFIX[layout]}.mp4`)
274
- run('ffmpeg', buildArgs({
275
- pic, voice: voicePath, plan, music: hasAudio(pic), subtitles: noCaptions ? null : burnIn(srtPath, layout),
276
- chapters, scoreLevel: p.cfg.scoreLevel ?? 0.55, dst
277
- }))
278
- log(`built ${p.rel(dst)} (${plan.rows.length} chapters)`)
279
- built.push((videos[layout] = dst))
280
- }
281
- rmSync(chapters, { force: true })
282
- if (videos.landscape) {
283
- const web = join(out, `${slug}-${locale}-web.mp4`)
284
- run('ffmpeg', webArgs(videos.landscape, web))
285
- log(`built ${p.rel(web)} (${webNote(statSync(web).size)})`)
286
- built.push(web)
287
- }
288
- for (const step of posterSteps({ render, out, slug, locale, built: videos, last: plan.rows.at(-1), exists: existsSync })) {
289
- if (step.copy) copyFileSync(...step.copy)
290
- else run('ffmpeg', step.ffmpeg)
291
- log(` ${p.rel((step.copy ?? step.ffmpeg).at(-1))}`)
292
- }
293
- log(`captions: ${p.rel(out)}/${slug}-${locale}.vtt/.srt; chapters.txt, description.txt`)
294
- return { built, duration: plan.total, plan }
295
- }