@senseihq/emaki-cli 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/src/build-media.js +166 -0
- package/src/build-video.js +113 -0
- package/src/campaign.js +2 -21
- package/src/capture.js +7 -155
- package/src/commands/campaign.js +63 -0
- package/src/commands/posting.js +74 -0
- package/src/commands/production.js +128 -0
- package/src/commands/run.js +19 -0
- package/src/commands/setup.js +97 -0
- package/src/csv.js +42 -0
- package/src/film-checks.js +102 -0
- package/src/films.js +3 -78
- package/src/fit.js +79 -0
- package/src/index.js +10 -342
- package/src/init.js +5 -9
- package/src/media.js +7 -163
- package/src/migrate.js +44 -27
- package/src/publish.js +43 -25
- package/src/render.js +125 -0
- package/src/show-page.js +75 -0
- package/src/stage.js +44 -31
- package/src/takes.js +18 -0
- package/src/tts-cloud.js +122 -0
- package/src/tts-io.js +51 -0
- package/src/tts-offline.js +85 -0
- package/src/tts.js +5 -237
- package/src/voice.js +10 -199
package/src/tts-io.js
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// What every text-to-speech provider shares: the TTSError a check or a synthesis throws, running a command, a take
|
|
2
|
+
// in the shape fit and build expect, keys from the environment, and the HTTP calls the cloud providers make.
|
|
3
|
+
import { spawnSync } from 'node:child_process'
|
|
4
|
+
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
|
5
|
+
import { tmpdir } from 'node:os'
|
|
6
|
+
import { join } from 'node:path'
|
|
7
|
+
|
|
8
|
+
export class TTSError extends Error {}
|
|
9
|
+
export const fail = (m) => { throw new TTSError(m) }
|
|
10
|
+
export const lang = (locale) => locale.split('-')[0]
|
|
11
|
+
|
|
12
|
+
export function sh(cmd, args, input) {
|
|
13
|
+
const r = spawnSync(cmd, args, { input, encoding: input == null ? 'utf-8' : undefined, maxBuffer: 256 * 1024 * 1024 })
|
|
14
|
+
if (r.error) fail(`${cmd}: ${r.error.code === 'ENOENT' ? 'not installed' : r.error.message}`)
|
|
15
|
+
if (r.status) fail(`${cmd}: ${String(r.stderr ?? '').slice(-400)}`)
|
|
16
|
+
return r.stdout
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** Any audio file or bytes → 48 kHz mono WAV with 0.1 s head air at dst. */
|
|
20
|
+
export function toTake(src, dst) {
|
|
21
|
+
const dir = mkdtempSync(join(tmpdir(), 'emaki-'))
|
|
22
|
+
try {
|
|
23
|
+
let file = src
|
|
24
|
+
if (typeof src !== 'string') { file = join(dir, 'raw'); writeFileSync(file, src) }
|
|
25
|
+
sh('ffmpeg', ['-v', 'error', '-y', '-i', file, '-af', 'adelay=100|100', '-ar', '48000', '-ac', '1', dst])
|
|
26
|
+
} finally {
|
|
27
|
+
rmSync(dir, { recursive: true, force: true })
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function inTemp(fn) {
|
|
32
|
+
const dir = mkdtempSync(join(tmpdir(), 'emaki-'))
|
|
33
|
+
try { return fn(dir) } finally { rmSync(dir, { recursive: true, force: true }) }
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function env(...names) {
|
|
37
|
+
for (const n of names) if (process.env[n]) return process.env[n]
|
|
38
|
+
return fail(`set ${names.join(' or ')} in the environment (never in files)`)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export async function post(url, body, headers, as = 'bytes') {
|
|
42
|
+
const r = await fetch(url, { method: 'POST', body: typeof body === 'string' || body instanceof Uint8Array ? body : JSON.stringify(body), headers })
|
|
43
|
+
if (!r.ok) fail(`${new URL(url).host}: HTTP ${r.status} ${(await r.text()).slice(0, 300)}`)
|
|
44
|
+
return as === 'json' ? r.json() : Buffer.from(await r.arrayBuffer())
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export async function get(url, headers) {
|
|
48
|
+
const r = await fetch(url, { headers })
|
|
49
|
+
if (!r.ok) fail(`${new URL(url).host}: HTTP ${r.status}`)
|
|
50
|
+
return r.json()
|
|
51
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
// The offline text-to-speech providers: Piper, Kokoro, macOS say and eSpeak NG. Each runs a local command and turns
|
|
2
|
+
// its output into a take (toTake). check(cfg, locale) throws TTSError when not ready; synth(text, dst, cfg, locale).
|
|
3
|
+
import { spawnSync } from 'node:child_process'
|
|
4
|
+
import { existsSync } from 'node:fs'
|
|
5
|
+
import { join } from 'node:path'
|
|
6
|
+
import { fail, inTemp, lang, sh, toTake } from './tts-io.js'
|
|
7
|
+
import { which } from './util.js'
|
|
8
|
+
|
|
9
|
+
const piper = {
|
|
10
|
+
label: 'Piper (offline)',
|
|
11
|
+
check(cfg) {
|
|
12
|
+
if (!which('piper')) fail('install it: pip install piper-tts')
|
|
13
|
+
if (!cfg.model || !existsSync(cfg.model)) fail(`voice model not found: "${cfg.model ?? ''}". Download a voice (.onnx and .onnx.json) from the Piper voices list and set "model"`)
|
|
14
|
+
},
|
|
15
|
+
async synth(text, dst, cfg) {
|
|
16
|
+
inTemp((d) => {
|
|
17
|
+
const raw = join(d, 'raw.wav')
|
|
18
|
+
const args = ['--model', cfg.model, '--output_file', raw]
|
|
19
|
+
if (cfg.speaker != null) args.push('--speaker', String(cfg.speaker))
|
|
20
|
+
if (cfg.length_scale) args.push('--length_scale', String(cfg.length_scale)) // >1 is slower
|
|
21
|
+
sh('piper', args, text)
|
|
22
|
+
if (!existsSync(raw)) fail('piper wrote nothing')
|
|
23
|
+
toTake(raw, dst)
|
|
24
|
+
})
|
|
25
|
+
},
|
|
26
|
+
voices: async () => ['Piper voices are files: pick one for this language from the Piper voice samples page, download its .onnx and .onnx.json, set "model".']
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const KOKORO_LANG = { en: 'a', 'en-GB': 'b', es: 'e', fr: 'f', hi: 'h', it: 'i', ja: 'j', pt: 'p', zh: 'z' }
|
|
30
|
+
const KOKORO_PY = `import sys, json, numpy as np, soundfile as sf
|
|
31
|
+
from kokoro import KPipeline
|
|
32
|
+
a = json.loads(sys.argv[1]); text = sys.stdin.read()
|
|
33
|
+
pipe = KPipeline(lang_code=a['code'])
|
|
34
|
+
audio = [x for _, _, x in pipe(text, voice=a['voice'], speed=a['speed'])]
|
|
35
|
+
sf.write(a['out'], np.concatenate(audio), 24000)`
|
|
36
|
+
const kokoro = {
|
|
37
|
+
label: 'Kokoro (offline)',
|
|
38
|
+
check(cfg, locale) {
|
|
39
|
+
const py = cfg.python ?? 'python3'
|
|
40
|
+
const r = spawnSync(py, ['-c', 'import kokoro, soundfile'], { encoding: 'utf-8' })
|
|
41
|
+
if (r.error || r.status) fail(`install it: ${py} -m pip install kokoro soundfile (and espeak-ng for some languages)`)
|
|
42
|
+
if (!KOKORO_LANG[locale] && !KOKORO_LANG[lang(locale)]) fail(`kokoro has no voices for ${locale}`)
|
|
43
|
+
},
|
|
44
|
+
async synth(text, dst, cfg, locale) {
|
|
45
|
+
inTemp((d) => {
|
|
46
|
+
const out = join(d, 'raw.wav')
|
|
47
|
+
const code = cfg.lang_code ?? KOKORO_LANG[locale] ?? KOKORO_LANG[lang(locale)]
|
|
48
|
+
sh(cfg.python ?? 'python3', ['-c', KOKORO_PY, JSON.stringify({ code, voice: cfg.voice ?? 'af_heart', speed: cfg.speed ?? 1, out })], text)
|
|
49
|
+
toTake(out, dst)
|
|
50
|
+
})
|
|
51
|
+
},
|
|
52
|
+
voices: async () => ['af_heart, af_bella, am_michael, bf_emma, bm_george (en); ef_dora, em_alex (es); ff_siwis (fr); if_sara (it); pf_dora (pt); jf_alpha (ja); zf_xiaobei (zh); see the Kokoro VOICES list']
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const macos = {
|
|
56
|
+
label: 'macOS say (offline, drafts)',
|
|
57
|
+
check() { if (!which('say')) fail('`say` exists only on macOS') },
|
|
58
|
+
async synth(text, dst, cfg) {
|
|
59
|
+
inTemp((d) => {
|
|
60
|
+
const raw = join(d, 'raw.aiff')
|
|
61
|
+
sh('say', ['-o', raw, ...(cfg.voice ? ['-v', cfg.voice] : []), ...(cfg.rate ? ['-r', String(cfg.rate)] : []), text])
|
|
62
|
+
toTake(raw, dst)
|
|
63
|
+
})
|
|
64
|
+
},
|
|
65
|
+
async voices(cfg, locale) {
|
|
66
|
+
const all = String(sh('say', ['-v', '?'])).split('\n').filter(Boolean)
|
|
67
|
+
const l = lang(locale), mine = all.filter((x) => x.includes(` ${l}_`) || x.includes(` ${l}-`))
|
|
68
|
+
return mine.length ? mine : all.slice(0, 40)
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const espeak = {
|
|
73
|
+
label: 'eSpeak NG (offline, robotic)',
|
|
74
|
+
check() { if (!which('espeak-ng')) fail('install espeak-ng (brew/apt/choco)') },
|
|
75
|
+
async synth(text, dst, cfg, locale) {
|
|
76
|
+
inTemp((d) => {
|
|
77
|
+
const raw = join(d, 'raw.wav')
|
|
78
|
+
sh('espeak-ng', ['-v', cfg.voice ?? locale, '-s', String(cfg.speed ?? 160), '-w', raw, text])
|
|
79
|
+
toTake(raw, dst)
|
|
80
|
+
})
|
|
81
|
+
},
|
|
82
|
+
voices: async (cfg, locale) => String(sh('espeak-ng', [`--voices=${lang(locale)}`])).split('\n').filter(Boolean)
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export { piper, kokoro, macos, espeak }
|
package/src/tts.js
CHANGED
|
@@ -20,246 +20,14 @@
|
|
|
20
20
|
// | google | cloud (Google) | GOOGLE_TTS_API_KEY, or `gcloud auth login` |
|
|
21
21
|
// | polly | cloud (AWS) | the `aws` CLI with credentials |
|
|
22
22
|
// | manual | a person | recorded take-NN.wav files |
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
26
|
-
import { join } from 'node:path'
|
|
27
|
-
import { which } from './util.js'
|
|
23
|
+
import { azure, elevenlabs, google, openai, polly } from './tts-cloud.js'
|
|
24
|
+
import { fail, lang, TTSError, toTake } from './tts-io.js'
|
|
25
|
+
import { espeak, kokoro, macos, piper } from './tts-offline.js'
|
|
28
26
|
|
|
29
|
-
export
|
|
30
|
-
const fail = (m) => { throw new TTSError(m) }
|
|
31
|
-
const lang = (locale) => locale.split('-')[0]
|
|
32
|
-
|
|
33
|
-
function sh(cmd, args, input) {
|
|
34
|
-
const r = spawnSync(cmd, args, { input, encoding: input == null ? 'utf-8' : undefined, maxBuffer: 256 * 1024 * 1024 })
|
|
35
|
-
if (r.error) fail(`${cmd}: ${r.error.code === 'ENOENT' ? 'not installed' : r.error.message}`)
|
|
36
|
-
if (r.status) fail(`${cmd}: ${String(r.stderr ?? '').slice(-400)}`)
|
|
37
|
-
return r.stdout
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
/** Any audio file or bytes → 48 kHz mono WAV with 0.1 s head air at dst. */
|
|
41
|
-
export function toTake(src, dst) {
|
|
42
|
-
const dir = mkdtempSync(join(tmpdir(), 'emaki-'))
|
|
43
|
-
try {
|
|
44
|
-
let file = src
|
|
45
|
-
if (typeof src !== 'string') { file = join(dir, 'raw'); writeFileSync(file, src) }
|
|
46
|
-
sh('ffmpeg', ['-v', 'error', '-y', '-i', file, '-af', 'adelay=100|100', '-ar', '48000', '-ac', '1', dst])
|
|
47
|
-
} finally {
|
|
48
|
-
rmSync(dir, { recursive: true, force: true })
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
function inTemp(fn) {
|
|
52
|
-
const dir = mkdtempSync(join(tmpdir(), 'emaki-'))
|
|
53
|
-
try { return fn(dir) } finally { rmSync(dir, { recursive: true, force: true }) }
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
function env(...names) {
|
|
57
|
-
for (const n of names) if (process.env[n]) return process.env[n]
|
|
58
|
-
return fail(`set ${names.join(' or ')} in the environment (never in files)`)
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
async function post(url, body, headers, as = 'bytes') {
|
|
62
|
-
const r = await fetch(url, { method: 'POST', body: typeof body === 'string' || body instanceof Uint8Array ? body : JSON.stringify(body), headers })
|
|
63
|
-
if (!r.ok) fail(`${new URL(url).host}: HTTP ${r.status} ${(await r.text()).slice(0, 300)}`)
|
|
64
|
-
return as === 'json' ? r.json() : Buffer.from(await r.arrayBuffer())
|
|
65
|
-
}
|
|
66
|
-
async function get(url, headers) {
|
|
67
|
-
const r = await fetch(url, { headers })
|
|
68
|
-
if (!r.ok) fail(`${new URL(url).host}: HTTP ${r.status}`)
|
|
69
|
-
return r.json()
|
|
70
|
-
}
|
|
27
|
+
export { TTSError, toTake }
|
|
71
28
|
|
|
72
29
|
// ── Providers: check(cfg, locale) throws TTSError when not ready; synth(text, dst, cfg, locale, ctx) ─────────────
|
|
73
|
-
|
|
74
|
-
const piper = {
|
|
75
|
-
label: 'Piper (offline)',
|
|
76
|
-
check(cfg) {
|
|
77
|
-
if (!which('piper')) fail('install it: pip install piper-tts')
|
|
78
|
-
if (!cfg.model || !existsSync(cfg.model)) fail(`voice model not found: "${cfg.model ?? ''}". Download a voice (.onnx and .onnx.json) from the Piper voices list and set "model"`)
|
|
79
|
-
},
|
|
80
|
-
async synth(text, dst, cfg) {
|
|
81
|
-
inTemp((d) => {
|
|
82
|
-
const raw = join(d, 'raw.wav')
|
|
83
|
-
const args = ['--model', cfg.model, '--output_file', raw]
|
|
84
|
-
if (cfg.speaker != null) args.push('--speaker', String(cfg.speaker))
|
|
85
|
-
if (cfg.length_scale) args.push('--length_scale', String(cfg.length_scale)) // >1 is slower
|
|
86
|
-
sh('piper', args, text)
|
|
87
|
-
if (!existsSync(raw)) fail('piper wrote nothing')
|
|
88
|
-
toTake(raw, dst)
|
|
89
|
-
})
|
|
90
|
-
},
|
|
91
|
-
voices: async () => ['Piper voices are files: pick one for this language from the Piper voice samples page, download its .onnx and .onnx.json, set "model".']
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
const KOKORO_LANG = { en: 'a', 'en-GB': 'b', es: 'e', fr: 'f', hi: 'h', it: 'i', ja: 'j', pt: 'p', zh: 'z' }
|
|
95
|
-
const KOKORO_PY = `import sys, json, numpy as np, soundfile as sf
|
|
96
|
-
from kokoro import KPipeline
|
|
97
|
-
a = json.loads(sys.argv[1]); text = sys.stdin.read()
|
|
98
|
-
pipe = KPipeline(lang_code=a['code'])
|
|
99
|
-
audio = [x for _, _, x in pipe(text, voice=a['voice'], speed=a['speed'])]
|
|
100
|
-
sf.write(a['out'], np.concatenate(audio), 24000)`
|
|
101
|
-
const kokoro = {
|
|
102
|
-
label: 'Kokoro (offline)',
|
|
103
|
-
check(cfg, locale) {
|
|
104
|
-
const py = cfg.python ?? 'python3'
|
|
105
|
-
const r = spawnSync(py, ['-c', 'import kokoro, soundfile'], { encoding: 'utf-8' })
|
|
106
|
-
if (r.error || r.status) fail(`install it: ${py} -m pip install kokoro soundfile (and espeak-ng for some languages)`)
|
|
107
|
-
if (!KOKORO_LANG[locale] && !KOKORO_LANG[lang(locale)]) fail(`kokoro has no voices for ${locale}`)
|
|
108
|
-
},
|
|
109
|
-
async synth(text, dst, cfg, locale) {
|
|
110
|
-
inTemp((d) => {
|
|
111
|
-
const out = join(d, 'raw.wav')
|
|
112
|
-
const code = cfg.lang_code ?? KOKORO_LANG[locale] ?? KOKORO_LANG[lang(locale)]
|
|
113
|
-
sh(cfg.python ?? 'python3', ['-c', KOKORO_PY, JSON.stringify({ code, voice: cfg.voice ?? 'af_heart', speed: cfg.speed ?? 1, out })], text)
|
|
114
|
-
toTake(out, dst)
|
|
115
|
-
})
|
|
116
|
-
},
|
|
117
|
-
voices: async () => ['af_heart, af_bella, am_michael, bf_emma, bm_george (en); ef_dora, em_alex (es); ff_siwis (fr); if_sara (it); pf_dora (pt); jf_alpha (ja); zf_xiaobei (zh); see the Kokoro VOICES list']
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
const macos = {
|
|
121
|
-
label: 'macOS say (offline, drafts)',
|
|
122
|
-
check() { if (!which('say')) fail('`say` exists only on macOS') },
|
|
123
|
-
async synth(text, dst, cfg) {
|
|
124
|
-
inTemp((d) => {
|
|
125
|
-
const raw = join(d, 'raw.aiff')
|
|
126
|
-
sh('say', ['-o', raw, ...(cfg.voice ? ['-v', cfg.voice] : []), ...(cfg.rate ? ['-r', String(cfg.rate)] : []), text])
|
|
127
|
-
toTake(raw, dst)
|
|
128
|
-
})
|
|
129
|
-
},
|
|
130
|
-
async voices(cfg, locale) {
|
|
131
|
-
const all = String(sh('say', ['-v', '?'])).split('\n').filter(Boolean)
|
|
132
|
-
const l = lang(locale), mine = all.filter((x) => x.includes(` ${l}_`) || x.includes(` ${l}-`))
|
|
133
|
-
return mine.length ? mine : all.slice(0, 40)
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
const espeak = {
|
|
138
|
-
label: 'eSpeak NG (offline, robotic)',
|
|
139
|
-
check() { if (!which('espeak-ng')) fail('install espeak-ng (brew/apt/choco)') },
|
|
140
|
-
async synth(text, dst, cfg, locale) {
|
|
141
|
-
inTemp((d) => {
|
|
142
|
-
const raw = join(d, 'raw.wav')
|
|
143
|
-
sh('espeak-ng', ['-v', cfg.voice ?? locale, '-s', String(cfg.speed ?? 160), '-w', raw, text])
|
|
144
|
-
toTake(raw, dst)
|
|
145
|
-
})
|
|
146
|
-
},
|
|
147
|
-
voices: async (cfg, locale) => String(sh('espeak-ng', [`--voices=${lang(locale)}`])).split('\n').filter(Boolean)
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
const elevenlabs = {
|
|
151
|
-
label: 'ElevenLabs',
|
|
152
|
-
check(cfg) {
|
|
153
|
-
env('ELEVENLABS_API_KEY')
|
|
154
|
-
if (!cfg.voice_id && !cfg.voice) fail('set "voice_id"')
|
|
155
|
-
},
|
|
156
|
-
async synth(text, dst, cfg, locale, ctx) {
|
|
157
|
-
const body = {
|
|
158
|
-
text, model_id: cfg.model ?? 'eleven_multilingual_v2', language_code: lang(locale),
|
|
159
|
-
voice_settings: { stability: cfg.stability ?? 0.55, similarity_boost: cfg.similarity ?? 0.8 }
|
|
160
|
-
}
|
|
161
|
-
if (ctx.prev) body.previous_text = ctx.prev
|
|
162
|
-
if (ctx.next) body.next_text = ctx.next
|
|
163
|
-
const audio = await post(`https://api.elevenlabs.io/v1/text-to-speech/${cfg.voice_id ?? cfg.voice}?output_format=mp3_44100_128`, body,
|
|
164
|
-
{ 'xi-api-key': env('ELEVENLABS_API_KEY'), 'content-type': 'application/json' })
|
|
165
|
-
toTake(audio, dst)
|
|
166
|
-
},
|
|
167
|
-
async voices() {
|
|
168
|
-
const data = await get('https://api.elevenlabs.io/v1/voices', { 'xi-api-key': env('ELEVENLABS_API_KEY') })
|
|
169
|
-
return (data.voices ?? []).map((v) => `${v.voice_id} ${v.name} ${JSON.stringify(v.labels ?? {})}`)
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
const openai = {
|
|
174
|
-
label: 'OpenAI (or Azure OpenAI)',
|
|
175
|
-
check() {
|
|
176
|
-
if (process.env.AZURE_OPENAI_ENDPOINT) env('AZURE_OPENAI_API_KEY')
|
|
177
|
-
else env('OPENAI_API_KEY')
|
|
178
|
-
},
|
|
179
|
-
async synth(text, dst, cfg) {
|
|
180
|
-
const body = { model: cfg.model ?? 'gpt-4o-mini-tts', voice: cfg.voice ?? 'alloy', input: text, response_format: 'wav' }
|
|
181
|
-
if (cfg.instructions) body.instructions = cfg.instructions // e.g. "Calm, warm, unhurried explainer."
|
|
182
|
-
let url, headers
|
|
183
|
-
if (process.env.AZURE_OPENAI_ENDPOINT) {
|
|
184
|
-
url = `${process.env.AZURE_OPENAI_ENDPOINT.replace(/\/$/, '')}/openai/deployments/${cfg.deployment ?? body.model}/audio/speech?api-version=${cfg.api_version ?? '2025-03-01-preview'}`
|
|
185
|
-
headers = { 'api-key': env('AZURE_OPENAI_API_KEY'), 'content-type': 'application/json' }
|
|
186
|
-
} else {
|
|
187
|
-
url = 'https://api.openai.com/v1/audio/speech'
|
|
188
|
-
headers = { authorization: `Bearer ${env('OPENAI_API_KEY')}`, 'content-type': 'application/json' }
|
|
189
|
-
}
|
|
190
|
-
toTake(await post(url, body, headers), dst)
|
|
191
|
-
},
|
|
192
|
-
voices: async () => ['alloy, ash, ballad, coral, echo, fable, nova, onyx, sage, shimmer, verse (all speak every supported language; set "instructions" for tone)']
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
const xml = (s) => s.replace(/[<>&'"]/g, (c) => ({ '<': '<', '>': '>', '&': '&', "'": ''', '"': '"' })[c])
|
|
196
|
-
const azure = {
|
|
197
|
-
label: 'Azure AI Speech',
|
|
198
|
-
check(cfg) {
|
|
199
|
-
env('AZURE_SPEECH_KEY'); env('AZURE_SPEECH_REGION')
|
|
200
|
-
if (!cfg.voice) fail('set "voice", e.g. "en-US-AvaMultilingualNeural" or "es-ES-ElviraNeural"')
|
|
201
|
-
},
|
|
202
|
-
async synth(text, dst, cfg, locale) {
|
|
203
|
-
const ssml = `<speak version='1.0' xml:lang='${locale}'><voice name='${cfg.voice}'><prosody rate='${cfg.rate ?? '0%'}'>${xml(text)}</prosody></voice></speak>`
|
|
204
|
-
const audio = await post(`https://${env('AZURE_SPEECH_REGION')}.tts.speech.microsoft.com/cognitiveservices/v1`, ssml, {
|
|
205
|
-
'Ocp-Apim-Subscription-Key': env('AZURE_SPEECH_KEY'), 'content-type': 'application/ssml+xml',
|
|
206
|
-
'X-Microsoft-OutputFormat': 'riff-48khz-16bit-mono-pcm', 'user-agent': 'emaki'
|
|
207
|
-
})
|
|
208
|
-
toTake(audio, dst)
|
|
209
|
-
},
|
|
210
|
-
async voices(cfg, locale) {
|
|
211
|
-
const data = await get(`https://${env('AZURE_SPEECH_REGION')}.tts.speech.microsoft.com/cognitiveservices/voices/list`, { 'Ocp-Apim-Subscription-Key': env('AZURE_SPEECH_KEY') })
|
|
212
|
-
return data.filter((v) => v.Locale.startsWith(locale) || v.Locale.split('-')[0] === lang(locale)).map((v) => `${v.ShortName} ${v.Gender}`)
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
function googleAuth() {
|
|
217
|
-
if (process.env.GOOGLE_TTS_API_KEY) return { 'X-Goog-Api-Key': process.env.GOOGLE_TTS_API_KEY }
|
|
218
|
-
if (which('gcloud')) {
|
|
219
|
-
const tok = spawnSync('gcloud', ['auth', 'print-access-token'], { encoding: 'utf-8' }).stdout?.trim()
|
|
220
|
-
if (tok) return { authorization: `Bearer ${tok}` }
|
|
221
|
-
}
|
|
222
|
-
return fail('set GOOGLE_TTS_API_KEY, or sign in with `gcloud auth login`')
|
|
223
|
-
}
|
|
224
|
-
const google = {
|
|
225
|
-
label: 'Google Cloud Text-to-Speech',
|
|
226
|
-
check(cfg) {
|
|
227
|
-
googleAuth()
|
|
228
|
-
if (!cfg.voice) fail('set "voice", e.g. "en-US-Neural2-F" or "es-ES-Chirp3-HD-Aoede"')
|
|
229
|
-
},
|
|
230
|
-
async synth(text, dst, cfg) {
|
|
231
|
-
const body = {
|
|
232
|
-
input: { text }, voice: { languageCode: cfg.voice.split('-').slice(0, 2).join('-'), name: cfg.voice },
|
|
233
|
-
audioConfig: { audioEncoding: 'LINEAR16', sampleRateHertz: 48000, speakingRate: cfg.speed ?? 1 }
|
|
234
|
-
}
|
|
235
|
-
const data = await post('https://texttospeech.googleapis.com/v1/text:synthesize', body, { ...googleAuth(), 'content-type': 'application/json' }, 'json')
|
|
236
|
-
toTake(Buffer.from(data.audioContent, 'base64'), dst)
|
|
237
|
-
},
|
|
238
|
-
async voices(cfg, locale) {
|
|
239
|
-
const data = await get(`https://texttospeech.googleapis.com/v1/voices?languageCode=${locale}`, googleAuth())
|
|
240
|
-
return (data.voices ?? []).map((v) => `${v.name} ${v.ssmlGender}`)
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
const polly = {
|
|
245
|
-
label: 'Amazon Polly',
|
|
246
|
-
check(cfg) {
|
|
247
|
-
if (!which('aws')) fail('install the AWS CLI and sign in (aws configure / SSO); credentials stay in your AWS profile')
|
|
248
|
-
if (!cfg.voice) fail('set "voice", e.g. "Joanna" (en-US) or "Lucia" (es-ES)')
|
|
249
|
-
},
|
|
250
|
-
async synth(text, dst, cfg, locale) {
|
|
251
|
-
inTemp((d) => {
|
|
252
|
-
const raw = join(d, 'raw.mp3')
|
|
253
|
-
sh('aws', ['polly', 'synthesize-speech', '--output-format', 'mp3', '--voice-id', cfg.voice, '--engine', cfg.engine ?? 'neural',
|
|
254
|
-
'--language-code', cfg.language ?? locale, '--text', text, ...(cfg.region ? ['--region', cfg.region] : []), raw])
|
|
255
|
-
toTake(raw, dst)
|
|
256
|
-
})
|
|
257
|
-
},
|
|
258
|
-
async voices(cfg, locale) {
|
|
259
|
-
const out = JSON.parse(String(sh('aws', ['polly', 'describe-voices', '--language-code', cfg.language ?? locale, '--engine', cfg.engine ?? 'neural', ...(cfg.region ? ['--region', cfg.region] : [])])))
|
|
260
|
-
return out.Voices.map((v) => `${v.Id} ${v.Gender}`)
|
|
261
|
-
}
|
|
262
|
-
}
|
|
30
|
+
// The offline ones are in tts-offline.js, the cloud ones in tts-cloud.js; manual needs nothing.
|
|
263
31
|
|
|
264
32
|
const manual = {
|
|
265
33
|
label: 'Recorded by a person',
|
package/src/voice.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
// The narration: choose a voice per locale
|
|
2
|
-
// (voice + score + captions, -14 LUFS).
|
|
3
|
-
import {
|
|
4
|
-
import { join
|
|
5
|
-
import { SUFFIX } from './config.js'
|
|
6
|
-
import {
|
|
7
|
-
buildArgs, buildPlan, chaptersText, description, ffmeta, planTable, posterSteps, storyLink, webArgs, webNote, webVoiceArgs, ytChapters
|
|
8
|
-
} from './media.js'
|
|
1
|
+
// The narration: choose a voice per locale and synthesise the takes. Fitting scene times to them (fit.js) and
|
|
2
|
+
// building the final videos (build-video.js: voice + score + captions, -14 LUFS) are re-exported from here.
|
|
3
|
+
import { existsSync, mkdirSync, writeFileSync } from 'node:fs'
|
|
4
|
+
import { join } from 'node:path'
|
|
9
5
|
import { OFFLINE, provider, TTSError, voiceConfig } from './tts.js'
|
|
10
|
-
import {
|
|
11
|
-
|
|
6
|
+
import { die, load, mediaLength, pad2, save } from './util.js'
|
|
7
|
+
|
|
8
|
+
// The steps after the takes, re-exported: the timing (fit.js), the final videos (build-video.js), the takes (takes.js).
|
|
9
|
+
export { fit, voiceWeb } from './fit.js'
|
|
10
|
+
export { build, captions } from './build-video.js'
|
|
11
|
+
export { takesFor } from './takes.js'
|
|
12
12
|
|
|
13
13
|
export const NO_PROVIDER = (loc) => `no voice provider is set for ${loc}. Choose one and record it with set-voice, for example:
|
|
14
14
|
offline piper (good, free) kokoro (very natural, free) macos (Mac drafts) espeak (robotic drafts)
|
|
@@ -104,192 +104,3 @@ export async function voice(p, slug, { locale, only, provider: override, opts =
|
|
|
104
104
|
writeFileSync(join(takes, 'provider.json'), JSON.stringify(keep, null, 1) + '\n')
|
|
105
105
|
return made
|
|
106
106
|
}
|
|
107
|
-
|
|
108
|
-
/** scene id → [take file, voice line] for the takes that exist. */
|
|
109
|
-
export function takesFor(p, slug, locale) {
|
|
110
|
-
const d = p.filmDir(slug), lines = load(join(d, 'voice', `${locale}.json`)), folder = p.prod(slug, locale, 'takes')
|
|
111
|
-
const out = new Map()
|
|
112
|
-
lines.forEach((l, i) => {
|
|
113
|
-
for (const ext of ['wav', 'mp3', 'm4a', 'aiff']) {
|
|
114
|
-
const f = join(folder, `take-${pad2(i + 1)}.${ext}`)
|
|
115
|
-
if (existsSync(f)) { out.set(l.scene, [f, l]); break }
|
|
116
|
-
}
|
|
117
|
-
})
|
|
118
|
-
return out
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
/** The narration for the browser: each take as voice/<locale>/<scene>.m4a beside the film, and in timing.<locale>.json
|
|
122
|
-
* each narrated scene's `voice` start (the same lead the videos use), so the story player and article figures speak
|
|
123
|
-
* in step with the picture. A scene whose take is gone loses its file and its start. Run after `fit`. */
|
|
124
|
-
export function voiceWeb(p, slug, { locale, encode = (src, dst) => run('ffmpeg', webVoiceArgs(src, dst)), log = () => {} } = {}) {
|
|
125
|
-
const d = p.filmDir(slug), takes = takesFor(p, slug, locale)
|
|
126
|
-
if (!takes.size) die(`${slug} has no takes for ${locale}: record or synthesise them first (emaki voice ${slug} --locale ${locale})`)
|
|
127
|
-
const film = load(join(d, 'film.json')), dir = join(d, 'voice', locale), tp = join(d, `timing.${locale}.json`)
|
|
128
|
-
const timing = existsSync(tp) ? load(tp) : { scenes: {} }
|
|
129
|
-
const tsc = (timing.scenes ??= {})
|
|
130
|
-
mkdirSync(dir, { recursive: true })
|
|
131
|
-
const wrote = [], removed = []
|
|
132
|
-
for (const s of film.scenes) {
|
|
133
|
-
const dst = join(dir, `${s.id}.m4a`), c = tsc[s.id] ?? {}
|
|
134
|
-
if (takes.has(s.id)) {
|
|
135
|
-
encode(takes.get(s.id)[0], dst)
|
|
136
|
-
wrote.push(p.rel(dst))
|
|
137
|
-
log(`${p.rel(dst)}`)
|
|
138
|
-
c.voice = p.lead
|
|
139
|
-
} else {
|
|
140
|
-
if (existsSync(dst)) { rmSync(dst); removed.push(p.rel(dst)) }
|
|
141
|
-
delete c.voice
|
|
142
|
-
}
|
|
143
|
-
if (Object.keys(c).length) tsc[s.id] = c
|
|
144
|
-
else delete tsc[s.id]
|
|
145
|
-
}
|
|
146
|
-
if (!readdirSync(dir).length) rmSync(dir, { recursive: true })
|
|
147
|
-
save(tp, timing)
|
|
148
|
-
return { wrote, removed, timing: p.rel(tp) }
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
const up = (v) => Math.ceil(v * 10 - 1e-9) / 10
|
|
152
|
-
|
|
153
|
-
/** When scene `s` stops moving, in scene seconds: film.json "settle", else the end of its JSON scene's tracks in this
|
|
154
|
-
* locale (word counts follow the translation), else null (a JS scene: its motion cannot be read). */
|
|
155
|
-
function settleOf(d, s, locale) {
|
|
156
|
-
if (s.settle != null) return s.settle
|
|
157
|
-
const f = join(d, `scene-${s.id}.json`)
|
|
158
|
-
if (!existsSync(f)) return null
|
|
159
|
-
const sp = join(d, `strings/${locale}.json`)
|
|
160
|
-
return motionEnd(localizeScene(load(f), existsSync(sp) ? load(sp) : {}))
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
/** timing.<locale>.json: a narrated scene runs lead + take + tail, longer or shorter than its design, but never ends
|
|
164
|
-
* before its motion settles; a scene without a take, or a JS scene without "settle", keeps its design time. */
|
|
165
|
-
export function fit(p, slug, { locale, check = false, length = mediaLength } = {}) {
|
|
166
|
-
const d = p.filmDir(slug), film = load(join(d, 'film.json')), takes = takesFor(p, slug, locale)
|
|
167
|
-
const tp = join(d, `timing.${locale}.json`)
|
|
168
|
-
const timing = existsSync(tp) ? load(tp) : { scenes: {} }
|
|
169
|
-
const tsc = (timing.scenes ??= {})
|
|
170
|
-
const rows = []
|
|
171
|
-
for (const s of film.scenes) {
|
|
172
|
-
const c = tsc[s.id] ?? {}
|
|
173
|
-
const holds = (c.holds ?? []).reduce((n, h) => n + h[1], 0)
|
|
174
|
-
const L = takes.has(s.id) ? length(takes.get(s.id)[0]) : 0
|
|
175
|
-
const base = s.dur + holds
|
|
176
|
-
let runs = base
|
|
177
|
-
if (L) {
|
|
178
|
-
const settle = settleOf(d, s, locale)
|
|
179
|
-
const floor = settle == null ? base : Math.min(base, up(settle + holds))
|
|
180
|
-
runs = Math.max(floor, up(p.lead + L + p.tail))
|
|
181
|
-
}
|
|
182
|
-
if (Math.abs(runs - base) > 1e-6) c.dur = runs
|
|
183
|
-
else delete c.dur
|
|
184
|
-
if (Object.keys(c).length) tsc[s.id] = c
|
|
185
|
-
else delete tsc[s.id]
|
|
186
|
-
rows.push({ scene: s.id, design: s.dur, holds, take: L, runs, longer: runs > base + 1e-6, shorter: runs < base - 1e-6 })
|
|
187
|
-
}
|
|
188
|
-
if (!check) save(tp, timing)
|
|
189
|
-
return { rows, total: rows.reduce((n, r) => n + r.runs, 0), path: p.rel(tp), written: !check }
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
const stamp = (t, sep = '.') => {
|
|
193
|
-
const h = Math.floor(t / 3600), m = Math.floor((t % 3600) / 60), s = t % 60
|
|
194
|
-
return `${pad2(h)}:${pad2(m)}:${s.toFixed(3).padStart(6, '0')}`.replace('.', sep)
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
/** Captions from the voice lines, spread over each take by sentence length. */
|
|
198
|
-
export function captions(takes, starts, lead, length = mediaLength) {
|
|
199
|
-
const vtt = ['WEBVTT', ''], srt = []
|
|
200
|
-
let k = 0
|
|
201
|
-
for (const [sid, [f, line]] of takes) {
|
|
202
|
-
let t0 = starts.get(sid)[0] + lead
|
|
203
|
-
const span = length(f) - 0.1
|
|
204
|
-
const parts = line.text.split(/(?<=[.!?¿¡;:])\s+/).map((x) => x.trim()).filter(Boolean)
|
|
205
|
-
const total = parts.reduce((n, x) => n + x.length, 0)
|
|
206
|
-
for (const x of parts) {
|
|
207
|
-
const dd = (span * x.length) / total
|
|
208
|
-
k++
|
|
209
|
-
vtt.push(`${stamp(t0)} --> ${stamp(t0 + dd)}`, x, '')
|
|
210
|
-
srt.push(String(k), `${stamp(t0, ',')} --> ${stamp(t0 + dd, ',')}`, x, '')
|
|
211
|
-
t0 += dd
|
|
212
|
-
}
|
|
213
|
-
}
|
|
214
|
-
return { vtt: vtt.join('\n'), srt: srt.join('\n') }
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
/** libass lays SRT out on a 384x288 canvas scaled to the video, so these are in 288ths of the height: landscape 16 ≈
|
|
218
|
-
* 60 px text 127 px up; portrait 9 ≈ 60 px text 333 px up (clear of the platform's buttons). */
|
|
219
|
-
const burnIn = (srtPath, layout) => {
|
|
220
|
-
const size = layout === 'landscape' ? 16 : 9, margin = layout === 'landscape' ? 34 : 50
|
|
221
|
-
return `subtitles=${srtPath.replace(/([:\\'])/g, '\\$1')}:force_style='FontName=Inter,FontSize=${size},PrimaryColour=&H00FFFFFF,OutlineColour=&H66000000,BackColour=&H66000000,BorderStyle=3,Outline=5,Shadow=0,MarginV=${margin}'`
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
/** Final videos per orientation (build.py): the rendered picture with the voice laid `lead` s into each scene, the
|
|
225
|
-
* render's music ducked under it, -14 LUFS, captions burned in (and shipped as .srt/.vtt). A take that outruns its
|
|
226
|
-
* scene freezes the scene's last frame until it has finished (buildPlan). `check` prints the plan and builds nothing;
|
|
227
|
-
* `out` is the output folder (default production/<slug>/<locale>/out/). */
|
|
228
|
-
export function build(p, slug, { locale, noCaptions = false, check = false, out: target = undefined, log = console.info, length = mediaLength } = {}) {
|
|
229
|
-
const d = p.filmDir(slug), film = load(join(d, 'film.json'))
|
|
230
|
-
const tp = join(d, `timing.${locale}.json`)
|
|
231
|
-
const tm = existsSync(tp) ? (load(tp).scenes ?? {}) : {}
|
|
232
|
-
const takes = takesFor(p, slug, locale)
|
|
233
|
-
const plan = buildPlan(film, tm, new Map([...takes].map(([sid, [file]]) => [sid, { file, seconds: length(file) }])), p)
|
|
234
|
-
const render = p.prod(slug, locale, 'render')
|
|
235
|
-
const pics = ['landscape', 'portrait'].map((layout) => [layout, join(render, `${slug}-${locale}-${SUFFIX[layout]}.mp4`)])
|
|
236
|
-
.filter(([, f]) => existsSync(f))
|
|
237
|
-
log(`${film.title ?? slug} (${locale}): ${plan.picture.toFixed(1)} s as rendered`)
|
|
238
|
-
planTable(plan).forEach((l) => log(l))
|
|
239
|
-
if (plan.missing.length) log(`missing takes: ${plan.missing.join(', ')}`)
|
|
240
|
-
if (check) {
|
|
241
|
-
if (!pics.length) log(`MISSING: no render in ${p.rel(render)}; run \`emaki render ${slug} --locale ${locale}\``)
|
|
242
|
-
return { built: [], duration: plan.total, plan }
|
|
243
|
-
}
|
|
244
|
-
if (!takes.size) die(`no takes in ${p.rel(p.prod(slug, locale, 'takes'))}; run \`emaki voice\` (or record them) first`)
|
|
245
|
-
if (!pics.length) die(`no render in ${p.rel(render)}; run \`emaki render ${slug} --locale ${locale}\` first`)
|
|
246
|
-
const out = target ? resolve(target) : p.prod(slug, locale, 'out')
|
|
247
|
-
mkdirSync(out, { recursive: true })
|
|
248
|
-
const voiced = new Map([...takes].filter(([sid]) => plan.rows.some((r) => r.id === sid)))
|
|
249
|
-
const starts = new Map(plan.rows.map((r) => [r.id, [r.at, r.tot]]))
|
|
250
|
-
const cap = captions(voiced, starts, p.lead, length)
|
|
251
|
-
const srtPath = join(out, `${slug}-${locale}.srt`)
|
|
252
|
-
writeFileSync(join(out, `${slug}-${locale}.vtt`), cap.vtt)
|
|
253
|
-
writeFileSync(srtPath, cap.srt)
|
|
254
|
-
const inputs = [], filters = []
|
|
255
|
-
;[...voiced].forEach(([sid, [f]], i) => {
|
|
256
|
-
inputs.push('-i', f)
|
|
257
|
-
const ms = Math.round((starts.get(sid)[0] + p.lead) * 1000)
|
|
258
|
-
filters.push(`[${i}:a]aresample=48000,adelay=${ms}|${ms},apad[v${i}]`)
|
|
259
|
-
})
|
|
260
|
-
const voicePath = join(out, 'voice.wav')
|
|
261
|
-
run('ffmpeg', ['-y', '-v', 'error', ...inputs, '-filter_complex',
|
|
262
|
-
filters.join(';') + ';' + [...voiced].map((_, i) => `[v${i}]`).join('') + `amix=inputs=${voiced.size}:normalize=0,atrim=0:${plan.total.toFixed(3)}[vo]`,
|
|
263
|
-
'-map', '[vo]', '-ac', '2', voicePath])
|
|
264
|
-
const chapters = join(out, 'chapters.ffmeta')
|
|
265
|
-
writeFileSync(chapters, ffmeta(plan.rows))
|
|
266
|
-
const yt = ytChapters(plan.rows)
|
|
267
|
-
writeFileSync(join(out, 'chapters.txt'), chaptersText(yt))
|
|
268
|
-
const story = join(p.content, 'stories', locale, `${slug}.json`)
|
|
269
|
-
const title = (existsSync(story) ? load(story).title : null) ?? film.title ?? slug
|
|
270
|
-
writeFileSync(join(out, 'description.txt'), description({ title, chapters: yt, link: storyLink(p.site, p.cfg, slug, locale) }))
|
|
271
|
-
const built = [], videos = {}
|
|
272
|
-
for (const [layout, pic] of pics) {
|
|
273
|
-
const dst = join(out, `${slug}-${locale}-${SUFFIX[layout]}.mp4`)
|
|
274
|
-
run('ffmpeg', buildArgs({
|
|
275
|
-
pic, voice: voicePath, plan, music: hasAudio(pic), subtitles: noCaptions ? null : burnIn(srtPath, layout),
|
|
276
|
-
chapters, scoreLevel: p.cfg.scoreLevel ?? 0.55, dst
|
|
277
|
-
}))
|
|
278
|
-
log(`built ${p.rel(dst)} (${plan.rows.length} chapters)`)
|
|
279
|
-
built.push((videos[layout] = dst))
|
|
280
|
-
}
|
|
281
|
-
rmSync(chapters, { force: true })
|
|
282
|
-
if (videos.landscape) {
|
|
283
|
-
const web = join(out, `${slug}-${locale}-web.mp4`)
|
|
284
|
-
run('ffmpeg', webArgs(videos.landscape, web))
|
|
285
|
-
log(`built ${p.rel(web)} (${webNote(statSync(web).size)})`)
|
|
286
|
-
built.push(web)
|
|
287
|
-
}
|
|
288
|
-
for (const step of posterSteps({ render, out, slug, locale, built: videos, last: plan.rows.at(-1), exists: existsSync })) {
|
|
289
|
-
if (step.copy) copyFileSync(...step.copy)
|
|
290
|
-
else run('ffmpeg', step.ffmpeg)
|
|
291
|
-
log(` ${p.rel((step.copy ?? step.ffmpeg).at(-1))}`)
|
|
292
|
-
}
|
|
293
|
-
log(`captions: ${p.rel(out)}/${slug}-${locale}.vtt/.srt; chapters.txt, description.txt`)
|
|
294
|
-
return { built, duration: plan.total, plan }
|
|
295
|
-
}
|