@mineo71/showtime 0.2.1 → 0.4.0

package/src/audio/decode.ts

@@ -0,0 +1,19 @@
+import { run, runBuffer } from '../ffmpeg';
+import { RATE } from './wav';
+
+const LOUDNORM = (lufs: number) => `loudnorm=I=${lufs}:TP=-1.5:LRA=11`;
+
+/** A synthesized line as a loudness-normalised mono 48 kHz 16-bit WAV. */
+export async function normalizeVoice(src: string, dest: string): Promise<void> {
+  await run('ffmpeg', ['-v', 'error', '-y', '-i', src, '-af', LOUDNORM(-16), '-ac', '1', '-ar', String(RATE), '-c:a', 'pcm_s16le', '-f', 'wav', dest]);
+}
+
+/** Any audio file ffmpeg reads, as float channels at 48 kHz (optionally loudness-normalised). */
+export async function decodeAudio(file: string, o: { channels: 1 | 2; lufs?: number }): Promise<Float32Array[]> {
+  const af = o.lufs === undefined ? [] : ['-af', LOUDNORM(o.lufs)];
+  const raw = await runBuffer('ffmpeg', ['-v', 'error', '-i', file, ...af, '-ac', String(o.channels), '-ar', String(RATE), '-f', 'f32le', '-']);
+  const n = Math.floor(raw.length / 4 / o.channels);
+  const channels = Array.from({ length: o.channels }, () => new Float32Array(n));
+  for (let i = 0; i < n; i++) for (let c = 0; c < o.channels; c++) channels[c][i] = raw.readFloatLE((i * o.channels + c) * 4);
+  return channels;
+}

package/src/audio/mix.ts

@@ -0,0 +1,72 @@
+// The soundtrack mixer: voices, a looped music bed that ducks under them, and click/key sounds.
+
+/** Music level while a voice plays. */
+export const DUCK = 0.3;
+export const DUCK_RAMP_MS = 300;
+export const MUSIC_FADE_IN_MS = 800;
+export const MUSIC_FADE_OUT_MS = 2000;
+/** Peak the finished track is scaled under when parts overlap. */
+const CEILING = 0.98;
+
+export interface MixPlan {
+  rate: number;
+  totalMs: number;
+  /** mono voice clips; `atMs` from the start of the track */
+  voices: { samples: Float32Array; atMs: number }[];
+  /** one or two channels, looped */
+  music?: { channels: Float32Array[]; volume: number };
+  sounds: { samples: Float32Array; atMs: number; gain: number }[];
+}
+
+const clamp01 = (v: number) => Math.min(1, Math.max(0, v));
+
+/** Music gain (before its volume) at `tMs`: fade in, fade out, and ducking around voice spans. */
+export function musicGain(tMs: number, spans: { startMs: number; endMs: number }[], totalMs: number): number {
+  const fade = Math.min(clamp01(tMs / MUSIC_FADE_IN_MS), clamp01((totalMs - tMs) / MUSIC_FADE_OUT_MS));
+  let duck = 0;
+  for (const s of spans) {
+    duck = Math.max(duck, clamp01(Math.min((tMs - (s.startMs - DUCK_RAMP_MS)) / DUCK_RAMP_MS, (s.endMs + DUCK_RAMP_MS - tMs) / DUCK_RAMP_MS)));
+  }
+  return fade * (1 - (1 - DUCK) * duck);
+}
+
+function addMono(into: Float32Array[], samples: Float32Array, at: number, gain: number) {
+  const end = Math.min(into[0].length, at + samples.length);
+  for (let i = Math.max(0, at); i < end; i++) {
+    const v = samples[i - at] * gain;
+    into[0][i] += v;
+    into[1][i] += v;
+  }
+}
+
+/** Mix the plan into a stereo track of `totalMs`. */
+export function mixTrack(plan: MixPlan): Float32Array[] {
+  const { rate } = plan;
+  const n = Math.round((plan.totalMs / 1000) * rate);
+  const out = [new Float32Array(n), new Float32Array(n)];
+  const at = (ms: number) => Math.round((ms / 1000) * rate);
+
+  if (plan.music && plan.music.channels[0]?.length) {
+    const src = plan.music.channels;
+    const len = src[0].length;
+    const spans = plan.voices.map((v) => ({ startMs: v.atMs, endMs: v.atMs + (v.samples.length / rate) * 1000 }));
+    // Gain per millisecond is plenty for ramps of hundreds of ms.
+    const gains = Float32Array.from({ length: Math.ceil(plan.totalMs) + 1 }, (_, ms) => musicGain(ms, spans, plan.totalMs) * plan.music!.volume);
+    for (let i = 0; i < n; i++) {
+      const g = gains[Math.floor((i / rate) * 1000)];
+      const j = i % len;
+      out[0][i] += src[0][j] * g;
+      out[1][i] += (src[1] ?? src[0])[j] * g;
+    }
+  }
+  for (const v of plan.voices) addMono(out, v.samples, at(v.atMs), 1);
+  for (const s of plan.sounds) addMono(out, s.samples, at(s.atMs), s.gain);
+
+  let peak = 0;
+  for (const ch of out) for (let i = 0; i < n; i++) peak = Math.max(peak, Math.abs(ch[i]));
+  if (peak > CEILING) {
+    const k = CEILING / peak;
+    for (const ch of out) for (let i = 0; i < n; i++) ch[i] *= k;
+  }
+  return out;
+}

package/src/audio/sfx.ts

@@ -0,0 +1,55 @@
+// Click and keystroke sounds, synthesized so the package ships no audio files.
+
+/** Small deterministic PRNG (mulberry32), so every render sounds the same. */
+function prng(seed: number) {
+  let a = seed >>> 0;
+  return () => {
+    a = (a + 0x6d2b79f5) >>> 0;
+    let t = a;
+    t = Math.imul(t ^ (t >>> 15), t | 1);
+    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+interface Voice {
+  ms: number;
+  seed: number;
+  noiseDecayMs: number;
+  tone: number;
+  toneDecayMs: number;
+  thump: number;
+  peak: number;
+}
+
+function synth(rate: number, v: Voice): Float32Array {
+  const n = Math.round((v.ms / 1000) * rate);
+  const out = new Float32Array(n);
+  const rand = prng(v.seed);
+  const fade = Math.round(0.005 * rate);
+  let low = 0;
+  let max = 0;
+  for (let i = 0; i < n; i++) {
+    const t = i / rate;
+    // A one-pole low-pass takes the hiss off the noise burst.
+    low += 0.35 * (rand() * 2 - 1 - low);
+    const s =
+      low * Math.exp(-t / (v.noiseDecayMs / 1000)) +
+      0.6 * Math.sin(2 * Math.PI * v.tone * t) * Math.exp(-t / (v.toneDecayMs / 1000)) +
+      v.thump * Math.sin(2 * Math.PI * 140 * t) * Math.exp(-t / 0.012);
+    out[i] = s * Math.min(1, (n - 1 - i) / fade);
+    max = Math.max(max, Math.abs(out[i]));
+  }
+  if (max > 0) for (let i = 0; i < n; i++) out[i] *= v.peak / max;
+  return out;
+}
+
+/** A mouse click: a soft tick with a little body, 45 ms. */
+export const clickSamples = (rate: number) =>
+  synth(rate, { ms: 45, seed: 7, noiseDecayMs: 4, tone: 1800, toneDecayMs: 9, thump: 0.5, peak: 0.9 });
+
+/** A keystroke: a lighter, shorter tick; variants differ a little so typing doesn't drone. */
+export const keySamples = (rate: number, variant: number) =>
+  synth(rate, { ms: 26, seed: 101 + variant, noiseDecayMs: 3, tone: 2600 + 350 * (variant % 4), toneDecayMs: 5, thump: 0.15, peak: 0.7 });
+
+export const KEY_VARIANTS = 4;

package/src/audio/soundtrack.ts

@@ -0,0 +1,49 @@
+import { mkdir } from 'node:fs/promises';
+import path from 'node:path';
+import { workPaths } from '../paths';
+import type { LoadedScript } from '../script/load';
+import { introFrames, totalFrames } from '../timeline/frames';
+import type { Timeline } from '../timeline/types';
+import { decodeAudio } from './decode';
+import { mixTrack } from './mix';
+import { clickSamples, KEY_VARIANTS, keySamples } from './sfx';
+import type { VoiceClip } from './tts';
+import { RATE, readWav, writeWav } from './wav';
+
+export const CLICK_GAIN = 0.5;
+export const KEY_GAIN = 0.2;
+/** Music is normalised to this loudness before its volume applies (voices sit at -16 LUFS). */
+const MUSIC_LUFS = -20;
+
+/**
+ * Mix narration, music and sounds into `.showtime/<name>/audio/mix.wav`, as long as the whole video
+ * (cards included). Returns its path relative to the renderer's public folder, or null when the
+ * video has no sound.
+ */
+export async function mixSoundtrack(loaded: LoadedScript, t: Timeline, clips: Record<string, VoiceClip>): Promise<string | null> {
+  const music = loaded.script.audio.music;
+  if (!t.narration.length && !t.sounds.length && !music) return null;
+  const frameMs = 1000 / t.fps;
+  const introMs = introFrames(t) * frameMs;
+  const voices = await Promise.all(
+    t.narration.map(async (n) => ({ samples: (await readWav(clips[n.where].file)).channels[0], atMs: introMs + n.atMs })),
+  );
+  const click = clickSamples(RATE);
+  const keys = Array.from({ length: KEY_VARIANTS }, (_, i) => keySamples(RATE, i));
+  const sounds = t.sounds.map((s, i) =>
+    s.kind === 'click'
+      ? { samples: click, atMs: introMs + s.atMs, gain: CLICK_GAIN }
+      : { samples: keys[i % KEY_VARIANTS], atMs: introMs + s.atMs, gain: KEY_GAIN },
+  );
+  const track = mixTrack({
+    rate: RATE,
+    totalMs: totalFrames(t) * frameMs,
+    voices,
+    sounds,
+    music: music ? { channels: await decodeAudio(music.file, { channels: 2, lufs: MUSIC_LUFS }), volume: music.volume } : undefined,
+  });
+  const paths = workPaths(loaded);
+  await mkdir(paths.audio, { recursive: true });
+  await writeWav(paths.mix, track);
+  return path.relative(paths.root, paths.mix).split(path.sep).join('/');
+}

package/src/audio/tts.ts

@@ -0,0 +1,186 @@
+import { spawn } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { existsSync } from 'node:fs';
+import { mkdir, rename, rm, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { workPaths } from '../paths';
+import type { LoadedScript } from '../script/load';
+import type { Voice } from '../script/schema';
+import { stepKey } from '../timeline/narration';
+import { normalizeVoice } from './decode';
+import { wavDurationMs } from './wav';
+
+export const TTS_TIMEOUT_MS = 120_000;
+/** `say` and `espeak-ng` speak at about this many words per minute at speed 1. */
+const BASE_WPM = 175;
+
+export interface EngineOptions {
+  /** Where API keys and base URLs come from (process.env by default). */
+  env?: NodeJS.ProcessEnv;
+  timeoutMs?: number;
+}
+
+export interface VoiceClip {
+  /** normalised mono 48 kHz WAV in the voice cache */
+  file: string;
+  durationMs: number;
+}
+
+/** Cache key of a line: every voice setting plus the text. */
+export function voiceKey(voice: Voice, text: string): string {
+  const parts = [voice.engine, voice.name ?? null, voice.speed, voice.model ?? null, voice.instructions ?? null, voice.command ?? null, text];
+  return createHash('sha256').update(JSON.stringify(parts)).digest('hex').slice(0, 16);
+}
+
+const tail = (s: string) => {
+  const t = s.trim().slice(-600);
+  return t ? `\n${t}` : '';
+};
+
+/** Run a program (or a shell command) with optional stdin; reject with the exit code and output tail. */
+function exec(cmd: string, args: string[], o: { input?: string; shell?: boolean; env?: NodeJS.ProcessEnv; timeoutMs: number }): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(cmd, args, { shell: o.shell ?? false, env: o.env, stdio: ['pipe', 'pipe', 'pipe'] });
+    let output = '';
+    const keep = (d: Buffer) => {
+      output = (output + d.toString()).slice(-4000);
+    };
+    child.stdout.on('data', keep);
+    child.stderr.on('data', keep);
+    const timer = setTimeout(() => {
+      child.kill('SIGKILL');
+      reject(new Error(`timed out after ${o.timeoutMs / 1000} s${tail(output)}`));
+    }, o.timeoutMs);
+    child.on('error', (err: NodeJS.ErrnoException) => {
+      clearTimeout(timer);
+      reject(err.code === 'ENOENT' ? new Error(`${cmd} not found`) : err);
+    });
+    child.on('close', (code) => {
+      clearTimeout(timer);
+      if (code === 0) resolve();
+      else reject(new Error(`exit code ${code}${tail(output)}`));
+    });
+    child.stdin.on('error', () => undefined); // a command that ignores stdin is fine
+    child.stdin.end(o.input ?? '');
+  });
+}
+
+const quote = (s: string) => (process.platform === 'win32' ? `"${s}"` : `'${s.replace(/'/g, `'\\''`)}'`);
+
+async function post(url: string, headers: Record<string, string>, body: unknown, out: string, timeoutMs: number): Promise<void> {
+  const res = await fetch(url, {
+    method: 'POST',
+    headers: { 'content-type': 'application/json', ...headers },
+    body: JSON.stringify(body),
+    signal: AbortSignal.timeout(timeoutMs),
+  });
+  if (!res.ok) throw new Error(`HTTP ${res.status}${tail(await res.text().catch(() => ''))}`);
+  await writeFile(out, Buffer.from(await res.arrayBuffer()));
+}
+
+function keyFrom(env: NodeJS.ProcessEnv, name: string): string {
+  const key = env[name];
+  if (!key) throw new Error(`${name} is not set. Export it in the shell that runs showtime.`);
+  return key;
+}
+
+/**
+ * Speak `text` with the voice's engine into `<base>.<ext>` and return that path. The text never
+ * reaches a command line: `say`/`espeak-ng` read it from a file, `command` from stdin.
+ */
+export async function synthesize(voice: Voice, text: string, base: string, opts: EngineOptions = {}): Promise<string> {
+  const env = opts.env ?? process.env;
+  const timeoutMs = opts.timeoutMs ?? TTS_TIMEOUT_MS;
+  switch (voice.engine) {
+    case 'system': {
+      const txt = `${base}.txt`;
+      await writeFile(txt, text);
+      const rate = voice.speed === 1 ? [] : [String(Math.round(BASE_WPM * voice.speed))];
+      if (process.platform === 'darwin') {
+        const out = `${base}.aiff`;
+        const args = ['-f', txt, '-o', out, ...(voice.name ? ['-v', voice.name] : []), ...(rate.length ? ['-r', ...rate] : [])];
+        await exec('say', args, { timeoutMs, env });
+        return out;
+      }
+      if (process.platform === 'linux') {
+        const out = `${base}.wav`;
+        const args = ['-f', txt, '-w', out, ...(voice.name ? ['-v', voice.name] : []), ...(rate.length ? ['-s', ...rate] : [])];
+        await exec('espeak-ng', args, { timeoutMs, env }).catch((err: Error) => {
+          throw new Error(`${err.message}. Install espeak-ng (apt install espeak-ng) or set audio.voice.engine to openai, elevenlabs or command.`);
+        });
+        return out;
+      }
+      throw new Error('the system voice works on macOS and Linux; set audio.voice.engine to openai, elevenlabs or command.');
+    }
+    case 'openai': {
+      const key = keyFrom(env, 'OPENAI_API_KEY');
+      const url = `${(env.OPENAI_BASE_URL ?? 'https://api.openai.com/v1').replace(/\/$/, '')}/audio/speech`;
+      const out = `${base}.wav`;
+      const body = {
+        model: voice.model ?? 'gpt-4o-mini-tts',
+        voice: voice.name ?? 'alloy',
+        input: text,
+        response_format: 'wav',
+        speed: voice.speed,
+        ...(voice.instructions ? { instructions: voice.instructions } : {}),
+      };
+      await post(url, { authorization: `Bearer ${key}` }, body, out, timeoutMs);
+      return out;
+    }
+    case 'elevenlabs': {
+      const key = keyFrom(env, 'ELEVENLABS_API_KEY');
+      const root = (env.ELEVENLABS_BASE_URL ?? 'https://api.elevenlabs.io').replace(/\/$/, '');
+      const url = `${root}/v1/text-to-speech/${encodeURIComponent(voice.name ?? '')}?output_format=mp3_44100_128`;
+      const out = `${base}.mp3`;
+      const body = { text, model_id: voice.model ?? 'eleven_multilingual_v2', voice_settings: { speed: Math.min(1.2, Math.max(0.7, voice.speed)) } };
+      await post(url, { 'xi-api-key': key }, body, out, timeoutMs);
+      return out;
+    }
+    case 'command': {
+      // .wav so tools that pick the format from the extension (say -o, sox) write something ffmpeg reads.
+      const out = `${base}.wav`;
+      const command = (voice.command ?? '').replaceAll('{out}', quote(out));
+      await exec(command, [], { shell: true, input: text, env: { ...env, SHOWTIME_OUT: out }, timeoutMs });
+      return out;
+    }
+  }
+}
+
+/**
+ * Synthesize every `show.say` line that is not cached yet, and return all lines by step. Lines are
+ * normalised (loudness, mono 48 kHz WAV) into `.showtime/<name>/audio/voice-<key>.wav`.
+ */
+export async function ensureVoices(
+  loaded: LoadedScript,
+  opts: EngineOptions & { log?: (line: string) => void } = {},
+): Promise<Record<string, VoiceClip>> {
+  const { voice } = loaded.script.audio;
+  const dir = workPaths(loaded).audio;
+  const clips: Record<string, VoiceClip> = {};
+  for (const scene of loaded.script.scenes) {
+    for (const [index, step] of scene.steps.entries()) {
+      const text = step.show?.say;
+      if (!text) continue;
+      const where = stepKey(scene.id, index);
+      const key = voiceKey(voice, text);
+      const file = path.join(dir, `voice-${key}.wav`);
+      if (!existsSync(file)) {
+        await mkdir(dir, { recursive: true });
+        opts.log?.(`Voice for ${where}…`);
+        const base = path.join(dir, `tmp-${key}`);
+        try {
+          const raw = await synthesize(voice, text, base, opts);
+          await normalizeVoice(raw, `${file}.part`);
+          await rename(`${file}.part`, file);
+        } catch (err) {
+          throw new Error(`audio.voice (${voice.engine}) failed for ${where}: ${(err as Error).message}`);
+        } finally {
+          for (const ext of ['.txt', '.aiff', '.wav', '.mp3']) await rm(`${base}${ext}`, { force: true });
+          await rm(`${file}.part`, { force: true });
+        }
+      }
+      clips[where] = { file, durationMs: await wavDurationMs(file) };
+    }
+  }
+  return clips;
+}

package/src/audio/wav.ts

@@ -0,0 +1,96 @@
+import { open, readFile, writeFile } from 'node:fs/promises';
+
+/** Sample rate of everything showTime mixes. */
+export const RATE = 48_000;
+const HEADER = 44;
+
+/** 16-bit PCM WAV from channels of float samples in [-1, 1] (out-of-range samples clip). */
+export function encodeWav(channels: Float32Array[], rate = RATE): Buffer {
+  const n = channels[0]?.length ?? 0;
+  const ch = channels.length;
+  const data = n * ch * 2;
+  const buf = Buffer.alloc(HEADER + data);
+  buf.write('RIFF', 0, 'ascii');
+  buf.writeUInt32LE(36 + data, 4);
+  buf.write('WAVE', 8, 'ascii');
+  buf.write('fmt ', 12, 'ascii');
+  buf.writeUInt32LE(16, 16);
+  buf.writeUInt16LE(1, 20); // PCM
+  buf.writeUInt16LE(ch, 22);
+  buf.writeUInt32LE(rate, 24);
+  buf.writeUInt32LE(rate * ch * 2, 28);
+  buf.writeUInt16LE(ch * 2, 32);
+  buf.writeUInt16LE(16, 34);
+  buf.write('data', 36, 'ascii');
+  buf.writeUInt32LE(data, 40);
+  let o = HEADER;
+  for (let i = 0; i < n; i++) {
+    for (let c = 0; c < ch; c++) {
+      const v = Math.max(-1, Math.min(1, channels[c][i]));
+      buf.writeInt16LE(Math.round(v * 32767), o);
+      o += 2;
+    }
+  }
+  return buf;
+}
+
+interface Format {
+  channels: number;
+  rate: number;
+  dataStart: number;
+  dataBytes: number;
+}
+
+/** Find the fmt and data chunks of a 16-bit PCM WAV. */
+function parseHeader(buf: Buffer): Format {
+  const bad = () => new Error('not a 16-bit PCM WAV file');
+  if (buf.length < 12 || buf.toString('ascii', 0, 4) !== 'RIFF' || buf.toString('ascii', 8, 12) !== 'WAVE') throw bad();
+  let o = 12;
+  let fmt: Omit<Format, 'dataStart' | 'dataBytes'> | null = null;
+  while (o + 8 <= buf.length) {
+    const id = buf.toString('ascii', o, o + 4);
+    const size = buf.readUInt32LE(o + 4);
+    if (id === 'fmt ') {
+      if (buf.readUInt16LE(o + 8) !== 1 || buf.readUInt16LE(o + 22) !== 16) throw bad();
+      fmt = { channels: buf.readUInt16LE(o + 10), rate: buf.readUInt32LE(o + 12) };
+    } else if (id === 'data') {
+      if (!fmt) throw bad();
+      return { ...fmt, dataStart: o + 8, dataBytes: Math.min(size, buf.length - o - 8) };
+    }
+    o += 8 + size + (size % 2);
+  }
+  throw bad();
+}
+
+export function decodeWav(buf: Buffer): { rate: number; channels: Float32Array[] } {
+  const f = parseHeader(buf);
+  const n = Math.floor(f.dataBytes / (2 * f.channels));
+  const channels = Array.from({ length: f.channels }, () => new Float32Array(n));
+  let o = f.dataStart;
+  for (let i = 0; i < n; i++) {
+    for (let c = 0; c < f.channels; c++) {
+      channels[c][i] = buf.readInt16LE(o) / 32767;
+      o += 2;
+    }
+  }
+  return { rate: f.rate, channels };
+}
+
+export const writeWav = (file: string, channels: Float32Array[], rate = RATE) => writeFile(file, encodeWav(channels, rate));
+
+export const readWav = async (file: string) => decodeWav(await readFile(file));
+
+/** Length of a WAV file from its header alone. */
+export async function wavDurationMs(file: string): Promise<number> {
+  const fh = await open(file, 'r');
+  try {
+    const head = Buffer.alloc(4096);
+    const { bytesRead } = await fh.read(head, 0, head.length, 0);
+    const f = parseHeader(head.subarray(0, bytesRead));
+    const { size } = await fh.stat();
+    const bytes = Math.min(size - f.dataStart, head.readUInt32LE(f.dataStart - 4));
+    return Math.round((bytes / (2 * f.channels) / f.rate) * 1000);
+  } finally {
+    await fh.close();
+  }
+}

package/src/cli/format.ts

@@ -1,4 +1,5 @@
 import path from 'node:path';
+import type { Issue } from '../lint';
 import type { CheckReport } from '../runner/check';
 
 const ARIA_LINES = 80;
@@ -17,3 +18,12 @@ export function formatCheck(report: CheckReport): string {
     ...aria.map((line) => `    ${line}`),
   ].join('\n');
 }
+
+export function formatLint(issues: Issue[], timing: boolean): string {
+  const lines = issues.map((i) => `${i.level === 'warn' ? '⚠' : 'ℹ'} ${i.where}  ${i.message}  [${i.rule}]`);
+  const warns = issues.filter((i) => i.level === 'warn').length;
+  const notes = issues.length - warns;
+  lines.push(issues.length ? `${warns} warning${warns === 1 ? '' : 's'}, ${notes} note${notes === 1 ? '' : 's'}` : '✓ no issues');
+  if (!timing) lines.push('(record a take to also check timing)');
+  return lines.join('\n');
+}