@hajimism/dek 0.1.0

package/src/voice/engine.ts

@@ -0,0 +1,177 @@
+import { DekError } from "../core/error.ts";
+
+export type VoiceSpeaker = {
+  name: string;
+  styles: Array<{ name: string; id: number }>;
+};
+
+export type AudioQuery = {
+  accent_phrases: Array<{
+    moras: Array<{ consonant_length?: number | null; vowel_length?: number | null }>;
+    pause_mora?: { vowel_length?: number | null } | null;
+  }>;
+  speedScale: number;
+  prePhonemeLength: number;
+  postPhonemeLength: number;
+  outputSamplingRate: number;
+  kana?: string;
+};
+
+const ENGINE_PORTS: Record<string, number> = {
+  voicevox: 50021,
+  aivis: 10101,
+  coeiroink: 50032,
+  sharevox: 50025,
+};
+
+export function engineBaseUrl(engine: string): string {
+  const override = process.env.DEK_VOICE_URL;
+  if (override) {
+    return override.replace(/\/$/, "");
+  }
+  if (/^https?:\/\//i.test(engine)) {
+    return engine.replace(/\/$/, "");
+  }
+  const [name, port] = engine.split(":");
+  const resolved = port ?? ENGINE_PORTS[name ?? ""] ?? ENGINE_PORTS.voicevox;
+  return `http://127.0.0.1:${resolved}`;
+}
+
+export async function fetchSpeakers(baseUrl: string): Promise<VoiceSpeaker[]> {
+  const res = await engineFetch(baseUrl, "/speakers");
+  return (await res.json()) as VoiceSpeaker[];
+}
+
+export async function fetchEngineVersion(baseUrl: string): Promise<string> {
+  const res = await engineFetch(baseUrl, "/version");
+  const text = (await res.text()).trim();
+  return text.replace(/^"|"$/g, "");
+}
+
+export async function fetchAudioQuery(
+  baseUrl: string,
+  text: string,
+  speaker: number,
+): Promise<AudioQuery> {
+  const res = await engineFetch(
+    baseUrl,
+    `/audio_query?text=${encodeURIComponent(text)}&speaker=${speaker}`,
+    { method: "POST" },
+  );
+  return (await res.json()) as AudioQuery;
+}
+
+export async function fetchSynthesis(
+  baseUrl: string,
+  query: AudioQuery,
+  speaker: number,
+): Promise<Buffer> {
+  const res = await engineFetch(baseUrl, `/synthesis?speaker=${speaker}`, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify(query),
+  });
+  return Buffer.from(await res.arrayBuffer());
+}
+
+export function resolveStyleId(speakers: VoiceSpeaker[], speaker: string): number {
+  const [name, styleName] = speaker.split("/");
+  const found = speakers.find((entry) => entry.name === name);
+  if (!found) {
+    throw new DekError(`speaker "${speaker}" not found`, {
+      hint: "run `dekc voice speakers`",
+    });
+  }
+  const style = found.styles.find((entry) => entry.name === (styleName ?? found.styles[0]?.name));
+  if (!style) {
+    throw new DekError(`style "${speaker}" not found`, {
+      hint: "run `dekc voice speakers`",
+    });
+  }
+  return style.id;
+}
+
+const ENGINE_SETUP: Record<string, { label: string; url: string; docker?: string }> = {
+  voicevox: {
+    label: "VOICEVOX",
+    url: "https://voicevox.hiroshiba.jp/",
+    docker: "docker run --rm -p 127.0.0.1:50021:50021 voicevox/voicevox_engine:cpu-latest",
+  },
+  aivis: {
+    label: "AivisSpeech",
+    url: "https://aivis-project.com/",
+    docker:
+      "docker run --rm -p 127.0.0.1:10101:10101 ghcr.io/aivis-project/aivisspeech-engine:cpu-latest",
+  },
+  coeiroink: { label: "COEIROINK", url: "https://coeiroink.com/" },
+  sharevox: { label: "SHAREVOX", url: "https://www.sharevox.app/" },
+};
+
+export function engineNameForUrl(baseUrl: string): string | undefined {
+  let port: string;
+  try {
+    port = new URL(baseUrl).port;
+  } catch {
+    return undefined;
+  }
+  return Object.keys(ENGINE_PORTS).find((name) => String(ENGINE_PORTS[name]) === port);
+}
+
+export function engineSetupHint(engine: string): string {
+  const fallback = "or point DEK_VOICE_URL or voice.toml engine at a running engine";
+  if (/^https?:\/\//i.test(engine)) {
+    return `start the engine at ${engine}, ${fallback}`;
+  }
+  const setup = ENGINE_SETUP[engine.split(":")[0] ?? ""];
+  if (!setup) {
+    return `start ${engine}, ${fallback}`;
+  }
+  const docker = setup.docker ? ` or run \`${setup.docker}\`` : "";
+  return `install ${setup.label} from ${setup.url}${docker}, ${fallback}`;
+}
+
+class EngineMissingError extends DekError {
+  constructor(engine: string, baseUrl: string) {
+    super(`${engine} was not found at ${baseUrl}`, { hint: engineSetupHint(engine) });
+    this.name = "DekError";
+  }
+}
+
+export function engineMissingError(engine: string, baseUrl: string): DekError {
+  return new EngineMissingError(engine, baseUrl);
+}
+
+/**
+ * Re-raises "engine is down" with the configured engine name so the hint
+ * names the right setup. An engine that is up but answers with an error is
+ * reported as it is; that is not an install problem.
+ */
+export async function withEngine<T>(
+  engine: string,
+  baseUrl: string,
+  fn: () => Promise<T>,
+): Promise<T> {
+  try {
+    return await fn();
+  } catch (error) {
+    if (error instanceof EngineMissingError) {
+      throw engineMissingError(engine, baseUrl);
+    }
+    throw error;
+  }
+}
+
+async function engineFetch(baseUrl: string, path: string, init?: RequestInit): Promise<Response> {
+  let res: Response;
+  try {
+    res = await fetch(`${baseUrl}${path}`, init);
+  } catch {
+    throw engineMissingError(engineNameForUrl(baseUrl) ?? "voice engine", baseUrl);
+  }
+  if (!res.ok) {
+    throw new DekError(`voice engine returned ${res.status} for ${path}`, {
+      hint: "check the engine log",
+    });
+  }
+  return res;
+}

package/src/voice/synth.ts

@@ -0,0 +1,127 @@
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { cuesFromDeck, splitSentences } from "../core/cue.ts";
+import { deckProjectRoot } from "../core/path.ts";
+import { asResolvedDeck, type ResolvedDeck, readDeckFile } from "../core/resolve.ts";
+import { isCachedFile, outputDir, replaceFile, requireInside } from "../core/safe-fs.ts";
+import { scheduleVoice, type Utterance } from "../core/timeline.ts";
+import {
+  loadVoiceDict,
+  loadVoiceSettings,
+  parseTimelineJson,
+  parseUtteranceJson,
+  resolveBeatTiming,
+  utteranceHash,
+  voiceCacheDir,
+  voiceCacheFile,
+  writeResolved,
+} from "../core/voice.ts";
+import {
+  engineBaseUrl,
+  fetchAudioQuery,
+  fetchEngineVersion,
+  fetchSpeakers,
+  fetchSynthesis,
+  resolveStyleId,
+  withEngine,
+} from "./engine.ts";
+import { concatWavs, silentWav, wavDurationMs } from "./wav.ts";
+
+export type SynthResult = {
+  timelinePath: string;
+  audioPath: string;
+  synthesized: number;
+  cached: number;
+};
+
+export async function synthDeck(dir: string): Promise<SynthResult>;
+export async function synthDeck(source: ResolvedDeck): Promise<SynthResult>;
+export async function synthDeck(input: string | ResolvedDeck): Promise<SynthResult> {
+  const { deck } = asResolvedDeck(input);
+  const root = deckProjectRoot(deck.dir);
+  const cacheDir = outputDir(voiceCacheDir(deck.dir), root);
+  const timelinePath = voiceCacheFile(deck.dir, "timeline.json");
+  const audioPath = voiceCacheFile(deck.dir, "audio.wav");
+  const pinTimeline = join(deck.dir, "voice", "pin", "timeline.json");
+  const pinAudio = join(deck.dir, "voice", "pin", "master.wav");
+  if (existsSync(pinTimeline) && existsSync(pinAudio)) {
+    const parsed = parseTimelineJson(readDeckFile(deck.dir, pinTimeline) ?? "", pinTimeline);
+    replaceFile(audioPath, readFileSync(requireInside(pinAudio, root)));
+    replaceFile(timelinePath, `${JSON.stringify({ ...parsed, audio: "audio.wav" }, null, 2)}\n`);
+    return { timelinePath, audioPath, synthesized: 0, cached: 0 };
+  }
+
+  const settings = loadVoiceSettings(deck.dir);
+  const dict = loadVoiceDict(deck.dir);
+  const baseUrl = engineBaseUrl(settings.engine);
+
+  const speakers = await withEngine(settings.engine, baseUrl, () => fetchSpeakers(baseUrl));
+  const version = await withEngine(settings.engine, baseUrl, () => fetchEngineVersion(baseUrl));
+
+  const styleId = resolveStyleId(speakers, settings.speaker);
+  writeResolved(deck.dir, {
+    styleId,
+    engineVersion: version,
+    speaker: settings.speaker,
+  });
+
+  const cues = cuesFromDeck(deck.deck, dict);
+  const utterances: Utterance[] = [];
+  const clips: Buffer[] = [];
+  let synthesized = 0;
+  let cached = 0;
+
+  const texts = cues.flatMap((cue) => cue.paragraphs.flatMap(splitSentences));
+  for (const text of texts) {
+    const hash = utteranceHash({
+      text,
+      engine: settings.engine,
+      engineVersion: version,
+      styleId,
+      speed: settings.speed,
+    });
+    const clipPath = join(cacheDir, `${hash}.wav`);
+    const metaPath = join(cacheDir, `${hash}.json`);
+    let utterance: Utterance | undefined;
+    let wav: Buffer | undefined;
+    if (isCachedFile(clipPath) && isCachedFile(metaPath)) {
+      utterance = parseUtteranceJson(readFileSync(metaPath, "utf8"));
+      if (utterance) {
+        wav = readFileSync(clipPath);
+        cached += 1;
+      }
+    }
+    if (!utterance || !wav) {
+      const query = await fetchAudioQuery(baseUrl, text, styleId);
+      query.speedScale = settings.speed;
+      wav = await fetchSynthesis(baseUrl, query, styleId);
+      utterance = {
+        text,
+        kana: query.kana ?? "",
+        durationMs: 0,
+      };
+      replaceFile(clipPath, wav);
+      synthesized += 1;
+    }
+    utterance = { ...utterance, text, durationMs: wavDurationMs(wav) };
+    replaceFile(metaPath, `${JSON.stringify(utterance)}\n`);
+    utterances.push(utterance);
+    clips.push(wav);
+  }
+
+  const scheduled = scheduleVoice(
+    cues,
+    utterances,
+    settings.pause,
+    "audio.wav",
+    resolveBeatTiming(deck.deck, settings).timing,
+  );
+  const audio =
+    clips.length > 0
+      ? concatWavs(clips, scheduled.pauseAfterMs, scheduled.leadingMs)
+      : silentWav(scheduled.timeline.durationMs);
+  replaceFile(audioPath, audio);
+  replaceFile(timelinePath, `${JSON.stringify(scheduled.timeline, null, 2)}\n`);
+
+  return { timelinePath, audioPath, synthesized, cached };
+}

package/src/voice/wav.ts

@@ -0,0 +1,115 @@
+export type PcmWav = {
+  sampleRate: number;
+  channels: number;
+  bitsPerSample: number;
+  pcm: Buffer;
+};
+
+export function encodeWav(wav: PcmWav): Buffer {
+  const blockAlign = (wav.channels * wav.bitsPerSample) / 8;
+  const byteRate = wav.sampleRate * blockAlign;
+  const header = Buffer.alloc(44);
+  header.write("RIFF", 0);
+  header.writeUInt32LE(36 + wav.pcm.length, 4);
+  header.write("WAVE", 8);
+  header.write("fmt ", 12);
+  header.writeUInt32LE(16, 16);
+  header.writeUInt16LE(1, 20);
+  header.writeUInt16LE(wav.channels, 22);
+  header.writeUInt32LE(wav.sampleRate, 24);
+  header.writeUInt32LE(byteRate, 28);
+  header.writeUInt16LE(blockAlign, 32);
+  header.writeUInt16LE(wav.bitsPerSample, 34);
+  header.write("data", 36);
+  header.writeUInt32LE(wav.pcm.length, 40);
+  return Buffer.concat([header, wav.pcm]);
+}
+
+export function parseWav(buf: Buffer): PcmWav {
+  if (buf.toString("ascii", 0, 4) !== "RIFF" || buf.toString("ascii", 8, 12) !== "WAVE") {
+    throw new Error("not a wav file");
+  }
+  let offset = 12;
+  let sampleRate = 24000;
+  let channels = 1;
+  let bitsPerSample = 16;
+  let pcm = Buffer.alloc(0);
+  while (offset + 8 <= buf.length) {
+    const id = buf.toString("ascii", offset, offset + 4);
+    const size = buf.readUInt32LE(offset + 4);
+    const start = offset + 8;
+    if (id === "fmt ") {
+      channels = buf.readUInt16LE(start + 2);
+      sampleRate = buf.readUInt32LE(start + 4);
+      bitsPerSample = buf.readUInt16LE(start + 14);
+    } else if (id === "data") {
+      pcm = Buffer.from(buf.subarray(start, start + size));
+      break;
+    }
+    offset = start + size + (size % 2);
+  }
+  return { sampleRate, channels, bitsPerSample, pcm };
+}
+
+export function silencePcm(durationMs: number, format: Omit<PcmWav, "pcm">): Buffer {
+  const bytesPerSample = (format.channels * format.bitsPerSample) / 8;
+  const samples = Math.max(0, Math.round((format.sampleRate * durationMs) / 1000));
+  return Buffer.alloc(samples * bytesPerSample);
+}
+
+export function wavDurationMs(buf: Buffer): number {
+  const wav = parseWav(buf);
+  const bytesPerSample = (wav.channels * wav.bitsPerSample) / 8;
+  if (bytesPerSample <= 0 || wav.sampleRate <= 0) {
+    return 0;
+  }
+  return Math.round((wav.pcm.length / bytesPerSample / wav.sampleRate) * 1000);
+}
+
+export function concatWavs(parts: Buffer[], pauseMs: number[], leadingMs = 0): Buffer {
+  if (parts.length === 0) {
+    return silentWav(Math.max(0, leadingMs));
+  }
+  const parsed = parts.map(parseWav);
+  const format = {
+    sampleRate: parsed[0]?.sampleRate ?? 24000,
+    channels: parsed[0]?.channels ?? 1,
+    bitsPerSample: parsed[0]?.bitsPerSample ?? 16,
+  };
+  for (const wav of parsed) {
+    if (
+      wav.sampleRate !== format.sampleRate ||
+      wav.channels !== format.channels ||
+      wav.bitsPerSample !== format.bitsPerSample
+    ) {
+      throw new Error("wav format mismatch");
+    }
+  }
+  const chunks: Buffer[] = [];
+  if (leadingMs > 0) {
+    chunks.push(silencePcm(leadingMs, format));
+  }
+  for (const [index, wav] of parsed.entries()) {
+    chunks.push(wav.pcm);
+    const pause = pauseMs[index];
+    if (pause && pause > 0) {
+      chunks.push(silencePcm(pause, format));
+    }
+  }
+  return encodeWav({ ...format, pcm: Buffer.concat(chunks) });
+}
+
+export function silentWav(durationMs: number): Buffer {
+  const format = { sampleRate: 24000, channels: 1, bitsPerSample: 16 };
+  return encodeWav({ ...format, pcm: silencePcm(durationMs, format) });
+}
+
+export function sliceWav(buf: Buffer, startMs: number, durationMs: number): Buffer {
+  const wav = parseWav(buf);
+  const bytesPerSample = (wav.channels * wav.bitsPerSample) / 8;
+  const startSample = Math.max(0, Math.round((wav.sampleRate * startMs) / 1000));
+  const durationSamples = Math.max(0, Math.round((wav.sampleRate * durationMs) / 1000));
+  const startByte = Math.min(wav.pcm.length, startSample * bytesPerSample);
+  const endByte = Math.min(wav.pcm.length, startByte + durationSamples * bytesPerSample);
+  return encodeWav({ ...wav, pcm: Buffer.from(wav.pcm.subarray(startByte, endByte)) });
+}