nixamp 0.53.5 → 0.55.0

package/src/ai-label.ts

@@ -33,6 +33,7 @@ import { dirname, join, resolve, sep } from "node:path";
 import { aiKey } from "./ai-keys.ts";
 import { composerFromEnv, OutOfCredit, type Composer } from "./ai-radio.ts";
 import { askJson, hasModel } from "./ai-text.ts";
+import { formatLrc, type SyncedLine } from "./lyrics.ts";
 import type { LibraryIndex, Probed } from "./library-index.ts";
 import type { StationRunner } from "./station-runner.ts";
 import { StationError, type StationStore } from "./stations.ts";
@@ -49,6 +50,8 @@ export interface LabelSong {
   /** The MP3, "" until composed. */
   file: string;
   lyricsFile: string;
+  /** Its words timed to the recording (001. <title>.lrc, enhanced LRC), for karaoke on the station page; absent until aligned. */
+  lrc?: string;
   /** Its music video (001. <title>.mp4), "" until made; absent on songs from before videos. */
   video?: string;
   /** Whether the video carries its song's tags (videos from 0.52.1 and before do not, until retagged). */
@@ -522,6 +525,55 @@ export async function workersAiCover(prompt: string, env: Record<string, string
   return image;
 }
 
+/** A timed word from an aligner: seconds from the start of the song. */
+export interface TimedWord { text: string; start: number; end: number }
+
+/** ElevenLabs forced alignment: the song and its words in, each word's time out. */
+export function elevenLabsAligner(key: string, fetcher: typeof fetch = fetch): (audio: string, text: string) => Promise<TimedWord[]> {
+  return async (audio, text) => {
+    const form = new FormData();
+    form.append("file", new Blob([readFileSync(audio)], { type: "audio/mpeg" }), "song.mp3");
+    form.append("text", text);
+    const response = await fetcher("https://api.elevenlabs.io/v1/forced-alignment", {
+      method: "POST",
+      headers: { "xi-api-key": key },
+      body: form,
+      signal: AbortSignal.timeout(5 * 60_000),
+    });
+    if (response.status === 401 || response.status === 402) throw new OutOfCredit(`elevenlabs answered HTTP ${response.status} to the alignment`);
+    if (!response.ok) throw new Error(`the alignment answered HTTP ${response.status}`);
+    const body = await response.json() as { words?: Array<{ text?: unknown; start?: unknown; end?: unknown }> };
+    return (body.words ?? [])
+      .map((one) => ({ text: String(one.text ?? "").trim(), start: Number(one.start), end: Number(one.end) }))
+      .filter((one) => one.text && Number.isFinite(one.start) && Number.isFinite(one.end));
+  };
+}
+
+/** The sung lines of a song's words: section tags ([Chorus]) and blank lines are not sung. */
+export function sungLines(lyrics: string): string[] {
+  return lyrics.split("\n").map((line) => line.trim()).filter((line) => line && !/^\[[^\]]*\]$/.test(line));
+}
+
+/**
+ * Timed words laid back onto the lines they were written in, in order, so
+ * each line and each word knows when it is sung.
+ */
+export function syncLines(lyrics: string, timed: TimedWord[]): SyncedLine[] {
+  const out: SyncedLine[] = [];
+  let at = 0;
+  for (const line of sungLines(lyrics)) {
+    const words = line.split(/\s+/).filter(Boolean);
+    const placed = words.map((word) => {
+      const hit = timed[at];
+      if (hit) at += 1;
+      return hit ? { text: word, start: hit.start, end: hit.end } : null;
+    }).filter((one): one is { text: string; start: number; end: number } => one !== null);
+    if (placed.length === 0) break;
+    out.push({ start: placed[0]!.start, end: placed[placed.length - 1]!.end, text: line, words: placed });
+  }
+  return out;
+}
+
 const STORYBOARD_SYSTEM = [
   "You storyboard music videos for original songs. Write one image prompt per scene, in order, following the song's story and mood.",
   "Keep the same main character(s), setting and color palette across scenes so it reads as one video; vary the shots (wide, close-up, action, detail).",
@@ -674,6 +726,8 @@ export interface AiLabelDeps {
   tag?: (from: string, to: string, tags: Record<string, string>, cover: string) => Promise<void>;
   /** Renders a song's music video to `out`; ffmpeg by default (renderVideo). */
   video?: (audio: string, cover: string, out: string) => Promise<void>;
+  /** Times each sung word in a recording; ElevenLabs forced alignment by default, null without a key. */
+  align?: ((audio: string, text: string) => Promise<TimedWord[]>) | null;
   /** Renders a storyboard music video; ffmpeg by default (renderStoryVideo). */
   storyVideo?: (input: { audio: string; images: string[]; seconds: number; artist: string; title: string; out: string }) => Promise<void>;
   /** A scene's picture for this prompt, or null when nothing can draw one; Workers AI by default. */
@@ -700,6 +754,8 @@ export class AiLabel {
   private filming: Promise<number> | null = null;
   /** Songs whose video failed since this server started. */
   private videoFailed = new Set<string>();
+  /** Songs whose words were already sent to the aligner since this server started. */
+  private alignTried = new Set<string>();
   /** Old videos already tried as storyboards since this server started. */
   private upgradeTried = new Set<string>();
   /** The day's album is tried at most every 30 minutes, so a failing model is not hammered. */
@@ -976,7 +1032,40 @@ export class AiLabel {
    * song's tags (seconds each), then old cover-and-waveform videos made again
    * as storyboards.
    */
+  /** The aligner this server can use: the dependency, else ElevenLabs with its key, else none. */
+  private aligner(): ((audio: string, text: string) => Promise<TimedWord[]>) | null {
+    if (this.deps.align !== undefined) return this.deps.align;
+    const key = aiKey("ELEVENLABS_API_KEY");
+    return key ? elevenLabsAligner(key) : null;
+  }
+
+  /** A song's words timed to its recording, written as 001. <title>.lrc beside it. Once per song (a failure is not retried this run). */
+  private async sync(artist: LabelArtist, album: LabelAlbum, song: LabelSong): Promise<boolean> {
+    const align = this.aligner();
+    if (!align || !song.file || !song.lyrics || song.lrc || this.alignTried.has(song.file)) return false;
+    this.alignTried.add(song.file);
+    try {
+      const timed = await align(song.file, sungLines(song.lyrics).join("\n"));
+      const lines = syncLines(song.lyrics, timed);
+      if (lines.length === 0) throw new Error("the aligner placed no words");
+      const file = join(album.folder, `${songBase(song)}.lrc`);
+      writeFileSync(file, formatLrc(lines, { title: song.title, artist: artist.name, album: album.title }));
+      song.lrc = file;
+      this.save();
+      return true;
+    } catch (error) {
+      this.log(`could not time the words of "${song.title}": ${(error as Error).message}`);
+      return false;
+    }
+  }
+
   private async filmPass(): Promise<number> {
+    // Words timed first: quick, and what the station page's karaoke needs.
+    for (const artist of this.list()) {
+      for (const album of artist.albums) {
+        for (const song of album.songs) await this.sync(artist, album, song);
+      }
+    }
     const songs = this.list().flatMap((artist) => artist.albums
       .filter((album) => album.cover || album.coverTried)
       .flatMap((album) => album.songs.filter((song) => song.file && !this.videoFailed.has(song.file)).map((song) => ({ artist, album, song }))));
@@ -1289,7 +1378,8 @@ export class AiLabel {
       this.save();
       this.writeAlbumJson(artist, album);
       await this.place(artist.genre);
-      // Its video is made beside it, without holding up the next song.
+      // Its words timed to the recording, then its video, without holding up the next song.
+      await this.sync(artist, album, song).catch(() => undefined);
       void this.videos();
     } catch (error) {
       this.failed(artist, error);
@@ -1465,6 +1555,7 @@ export class AiLabel {
       for (const song of songs) {
         entries.push({ name: `${folder}/${songBase(song)}.mp3`, path: song.file });
         entries.push({ name: `${folder}/${songBase(song)}.lyrics.txt`, bytes: Buffer.from(`${song.lyrics}\n`) });
+        if (song.lrc) entries.push({ name: `${folder}/${songBase(song)}.lrc`, path: song.lrc });
         if (song.video) entries.push({ name: `${folder}/${songBase(song)}.mp4`, path: song.video });
       }
       if (album.cover) entries.push({ name: `${folder}/cover.jpg`, path: album.cover });

package/src/follow-page.ts

@@ -0,0 +1,127 @@
+/**
+ * /follow on nixamp.com: follow an artist from a /radio page (radio-pages.ts).
+ *
+ * The /radio pages are on another site (live.streaming.pit.world), where a
+ * nixamp.com session cookie is a third-party cookie and no browser sends it.
+ * So Follow there is a link here, where the session is first-party: this page
+ * says who you are following, and a same-site form (SameSite=Lax keeps a
+ * cross-site POST from carrying the session) follows or unfollows. The follow
+ * is the same follows row as following a streamer, keyed "artist:<name>".
+ *
+ *   GET  /follow?id=artist:<name>&name=<shown>&back=<https page>
+ *   POST /follow   id, name, back, act=follow|unfollow  -> 303 back to the GET
+ */
+import { createHash } from "node:crypto";
+import type { IncomingMessage, ServerResponse } from "node:http";
+
+export interface FollowPageContext {
+  /** The signed-in account, or null. */
+  account: () => Promise<{ id: string } | null>;
+  follows: {
+    follow: (follower: string, streamer: string) => Promise<boolean>;
+    unfollow: (follower: string, streamer: string) => Promise<void>;
+    isFollowing: (follower: string, streamer: string) => Promise<boolean>;
+    followerCount: (streamer: string) => Promise<number>;
+  };
+  readBody: (request: IncomingMessage) => Promise<string>;
+}
+
+const ID = /^artist:[a-z0-9-]{1,90}$/;
+
+function escape(text: string): string {
+  return String(text).replace(/[&<>"']/g, (one) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" })[one]!);
+}
+
+const STYLE = `
+:root{--bg:#080c09;--fg:#e8efe9;--muted:#9aa79d;--card:#0e1410;--line:#1f2a22;--accent:#3ddc84}
+*{box-sizing:border-box}html,body{margin:0;background:var(--bg);color:var(--fg);font:16px/1.5 system-ui,-apple-system,"Segoe UI",sans-serif}
+main{max-width:520px;margin:0 auto;padding:40px 16px}
+.card{background:var(--card);border:1px solid var(--line);border-radius:16px;padding:24px}
+h1{margin:0 0 4px;font-size:1.5rem;overflow-wrap:anywhere}.by{color:var(--muted);margin:0 0 16px}
+button,a.go{display:block;text-align:center;text-decoration:none;margin-top:18px;width:100%;padding:12px;border:0;border-radius:999px;background:var(--accent);color:#06120a;font:inherit;font-weight:800;cursor:pointer}
+button.quiet{background:transparent;color:var(--fg);border:1px solid var(--line)}
+a{color:var(--accent)}a:focus-visible,button:focus-visible{outline:3px solid var(--accent);outline-offset:2px}
+small{display:block;margin-top:18px;color:var(--muted)}
+`;
+
+const hash = (text: string): string => `'sha256-${createHash("sha256").update(text).digest("base64")}'`;
+
+export const FOLLOW_CSP = ["default-src 'none'", `style-src ${hash(STYLE)}`, "img-src 'self'", "form-action 'self'", "base-uri 'none'", "frame-ancestors 'none'"].join("; ");
+
+function page(title: string, body: string): string {
+  return `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
+<title>${escape(title)} · nixamp</title><meta name="robots" content="noindex"><link rel="icon" href="/favicon.png" type="image/png"><style>${STYLE}</style></head>
+<body><main><div class="card">${body}</div></main></body></html>`;
+}
+
+function send(response: ServerResponse, status: number, html: string, method: string): void {
+  response.writeHead(status, { "content-type": "text/html; charset=utf-8", "content-security-policy": FOLLOW_CSP, "cache-control": "no-store", "referrer-policy": "no-referrer", "x-content-type-options": "nosniff" });
+  response.end(method === "HEAD" ? undefined : html);
+}
+
+/** What the link says, cleaned: an artist id, a name to show, and an https page to go back to. */
+export function followTarget(input: URLSearchParams): { id: string; name: string; back: string } | null {
+  const id = (input.get("id") ?? "").trim();
+  if (!ID.test(id)) return null;
+  const name = (input.get("name") ?? "").replace(/[\u0000-\u001f]/g, " ").trim().slice(0, 120) || id.slice("artist:".length);
+  let back = "";
+  try {
+    const url = new URL(input.get("back") ?? "");
+    if (url.protocol === "https:" && !url.username && !url.password) back = url.href;
+  } catch {
+    back = "";
+  }
+  return { id, name, back };
+}
+
+/** Answers /follow; false for any other path. */
+export async function handleFollowPage(request: IncomingMessage, response: ServerResponse, path: string, url: URL, ctx: FollowPageContext): Promise<boolean> {
+  if (path !== "/follow") return false;
+  const method = request.method ?? "GET";
+  if (method === "POST") {
+    // Same site only: a form on another site gets no session (SameSite=Lax), and is told so here too.
+    const origin = String(request.headers.origin ?? "");
+    const host = String(request.headers["x-forwarded-host"] ?? request.headers.host ?? "").split(",")[0]!.trim();
+    if (origin && origin !== "null" && new URL(origin).host !== host) {
+      send(response, 403, page("Not from here", `<h1>Not from here</h1><p class="by">Follow from nixamp.com's own page.</p>`), method);
+      return true;
+    }
+    const form = new URLSearchParams(await ctx.readBody(request));
+    const target = followTarget(form);
+    const me = await ctx.account().catch(() => null);
+    if (target && me) {
+      if (form.get("act") === "unfollow") await ctx.follows.unfollow(me.id, target.id);
+      else await ctx.follows.follow(me.id, target.id);
+    }
+    const again = target ? `/follow?${new URLSearchParams({ id: target.id, name: target.name, ...(target.back ? { back: target.back } : {}) }).toString()}` : "/follow";
+    response.writeHead(303, { location: again, "cache-control": "no-store" });
+    response.end();
+    return true;
+  }
+  if (method !== "GET" && method !== "HEAD") {
+    response.writeHead(405, { allow: "GET, POST" }).end();
+    return true;
+  }
+  const target = followTarget(url.searchParams);
+  if (!target) {
+    send(response, 404, page("Nobody to follow", `<h1>Nobody to follow</h1><p class="by">That link does not name an artist.</p>`), method);
+    return true;
+  }
+  const back = target.back ? `<small><a href="${escape(target.back)}">← Back to ${escape(target.name)}</a></small>` : "";
+  const me = await ctx.account().catch(() => null);
+  if (!me) {
+    send(response, 200, page(`Follow ${target.name}`, `<h1>Follow ${escape(target.name)}</h1>
+<p class="by">Sign in to nixamp.com to follow the artists you hear on the radio.</p>
+<a class="go" href="/radio#music">Sign in to nixamp.com</a>
+<small>Signed in, open this page again (or press Follow on the artist's page) to follow.</small>${back}`), method);
+    return true;
+  }
+  const [following, count] = await Promise.all([ctx.follows.isFollowing(me.id, target.id), ctx.follows.followerCount(target.id).catch(() => 0)]);
+  const hidden = `<input type="hidden" name="id" value="${escape(target.id)}"><input type="hidden" name="name" value="${escape(target.name)}">${target.back ? `<input type="hidden" name="back" value="${escape(target.back)}">` : ""}`;
+  send(response, 200, page(`Follow ${target.name}`, `<h1>${escape(target.name)}</h1>
+<p class="by">${following ? "You follow them." : "Follow them to keep them with your follows on nixamp.com."} ${count} follower${count === 1 ? "" : "s"}.</p>
+<form method="post" action="/follow">${hidden}<input type="hidden" name="act" value="${following ? "unfollow" : "follow"}">
+<button type="submit"${following ? ' class="quiet"' : ""}>${following ? "Unfollow" : `Follow ${escape(target.name)}`}</button></form>
+${back}`), method);
+  return true;
+}

package/src/lyrics.ts

@@ -13,6 +13,7 @@
  * nichedb's for ten minutes, misses included, so a station of tracks nobody
  * has words for asks once per track per ten minutes and never waits long.
  */
+import { readFileSync } from "node:fs";
 
 export interface Lyrics {
   title: string;
@@ -22,6 +23,76 @@ export interface Lyrics {
   source: string;
   /** ISO 639-1 where we know the mapping, else what the tag said; "" when unsaid. */
   language: string;
+  /** When a .lrc beside the file says when each line (and word) is sung: seconds from the file's start. */
+  synced?: SyncedLine[];
+}
+
+export interface SyncedLine {
+  start: number;
+  end: number;
+  text: string;
+  words: Array<{ start: number; end: number; text: string }>;
+}
+
+const STAMP = /^(\d{1,3}):(\d{2})(?:[.:](\d{1,3}))?$/;
+
+function seconds(stamp: string): number {
+  const match = STAMP.exec(stamp);
+  if (!match) return Number.NaN;
+  const fraction = match[3] ? Number(`0.${match[3]}`) : 0;
+  return Number(match[1]) * 60 + Number(match[2]) + fraction;
+}
+
+function stamp(at: number): string {
+  const whole = Math.max(0, at);
+  const minutes = Math.floor(whole / 60);
+  const rest = whole - minutes * 60;
+  return `${String(minutes).padStart(2, "0")}:${rest.toFixed(2).padStart(5, "0")}`;
+}
+
+/**
+ * Enhanced LRC: `[mm:ss.xx]` before each sung line, and `<mm:ss.xx>` before
+ * each word when the words are timed. Tags like `[ar:...]` and lines without a
+ * time are skipped. Ends are the next start (the last line runs 4 s).
+ */
+export function parseLrc(text: string): SyncedLine[] {
+  const lines: SyncedLine[] = [];
+  for (const raw of text.replace(/\r\n?/g, "\n").split("\n")) {
+    const head = /^\[([^\]]+)\](.*)$/.exec(raw.trim());
+    if (!head) continue;
+    const start = seconds(head[1]!);
+    if (!Number.isFinite(start)) continue;
+    const body = head[2]!;
+    const words: SyncedLine["words"] = [];
+    const parts = body.split(/<([^>]+)>/);
+    if (parts.length > 1) {
+      for (let i = 1; i < parts.length; i += 2) {
+        const at = seconds(parts[i]!);
+        const word = (parts[i + 1] ?? "").trim();
+        if (Number.isFinite(at) && word) words.push({ start: at, end: at, text: word });
+      }
+    }
+    const plain = (parts.length > 1 ? words.map((one) => one.text).join(" ") : body).trim();
+    if (!plain) continue;
+    lines.push({ start, end: start, text: plain, words });
+  }
+  lines.sort((a, b) => a.start - b.start);
+  lines.forEach((line, i) => {
+    line.end = lines[i + 1]?.start ?? line.start + 4;
+    line.words.forEach((word, j) => { word.end = line.words[j + 1]?.start ?? line.end; });
+  });
+  return lines;
+}
+
+export function formatLrc(lines: SyncedLine[], head: { title?: string; artist?: string; album?: string } = {}): string {
+  const tags = [head.artist ? `[ar:${head.artist}]` : "", head.title ? `[ti:${head.title}]` : "", head.album ? `[al:${head.album}]` : ""].filter(Boolean);
+  const body = lines.map((line) => `[${stamp(line.start)}]${line.words.length ? line.words.map((word) => ` <${stamp(word.start)}>${word.text}`).join("") : line.text}`);
+  return `${[...tags, ...body].join("\n")}\n`;
+}
+
+/** The .lrc that sits beside a file (same name, .lrc), or "". */
+export function lrcBeside(file: string): string {
+  return file.replace(/\.[^./]+$/, "") + ".lrc";
 }
 
 export const NICHEDB_LYRICS = "https://nichedb.dev";
@@ -116,7 +187,11 @@ export class LyricsFinder {
     const title = tags["title"] || known?.title || "";
     const artist = tags["artist"] || tags["album_artist"] || known?.artist || "";
     const own = lyricsFromTags(tags);
-    if (own) return { title, artist, text: own.text, source: "embedded", language: own.language };
+    // Timed words beside the file (karaoke): the page follows along with them.
+    let synced: SyncedLine[] = [];
+    try { synced = parseLrc(readFileSync(lrcBeside(file), "utf8")); } catch { synced = []; }
+    if (own) return { title, artist, text: own.text, source: "embedded", language: own.language, ...(synced.length ? { synced } : {}) };
+    if (synced.length) return { title, artist, text: synced.map((line) => line.text).join("\n"), source: "embedded", language: "", synced };
     if (!title || !artist || !this.base) return null;
     return this.lookup(artist, title);
   }