nixamp 0.53.5 → 0.55.0

package/dist/ai-label.d.ts

@@ -1,5 +1,6 @@
 import { type Composer } from "./ai-radio.ts";
 import { askJson } from "./ai-text.ts";
+import { type SyncedLine } from "./lyrics.ts";
 import type { LibraryIndex, Probed } from "./library-index.ts";
 import type { StationRunner } from "./station-runner.ts";
 import { type StationStore } from "./stations.ts";
@@ -15,6 +16,8 @@ export interface LabelSong {
     /** The MP3, "" until composed. */
     file: string;
     lyricsFile: string;
+    /** Its words timed to the recording (001. <title>.lrc, enhanced LRC), for karaoke on the station page; absent until aligned. */
+    lrc?: string;
     /** Its music video (001. <title>.mp4), "" until made; absent on songs from before videos. */
     video?: string;
     /** Whether the video carries its song's tags (videos from 0.52.1 and before do not, until retagged). */
@@ -127,6 +130,21 @@ export declare function writeLyrics(artist: Pick<LabelArtist, "name" | "genre" |
  * OPENAI_API_KEY its token); null when there is no image model to ask.
  */
 export declare function workersAiCover(prompt: string, env?: Record<string, string | undefined>, fetcher?: typeof fetch): Promise<Buffer | null>;
+/** A timed word from an aligner: seconds from the start of the song. */
+export interface TimedWord {
+    text: string;
+    start: number;
+    end: number;
+}
+/** ElevenLabs forced alignment: the song and its words in, each word's time out. */
+export declare function elevenLabsAligner(key: string, fetcher?: typeof fetch): (audio: string, text: string) => Promise<TimedWord[]>;
+/** The sung lines of a song's words: section tags ([Chorus]) and blank lines are not sung. */
+export declare function sungLines(lyrics: string): string[];
+/**
+ * Timed words laid back onto the lines they were written in, in order, so
+ * each line and each word knows when it is sung.
+ */
+export declare function syncLines(lyrics: string, timed: TimedWord[]): SyncedLine[];
 /** Image prompts for a song's video, one a scene, from its words and mood. */
 export declare function writeStoryboard(artist: Pick<LabelArtist, "name" | "genre" | "sound">, album: Pick<LabelAlbum, "title" | "concept">, song: Pick<LabelSong, "title" | "style" | "lyrics">, scenes: number, ask?: typeof askJson): Promise<string[]>;
 /**
@@ -169,6 +187,8 @@ export interface AiLabelDeps {
     tag?: (from: string, to: string, tags: Record<string, string>, cover: string) => Promise<void>;
     /** Renders a song's music video to `out`; ffmpeg by default (renderVideo). */
     video?: (audio: string, cover: string, out: string) => Promise<void>;
+    /** Times each sung word in a recording; ElevenLabs forced alignment by default, null without a key. */
+    align?: ((audio: string, text: string) => Promise<TimedWord[]>) | null;
     /** Renders a storyboard music video; ffmpeg by default (renderStoryVideo). */
     storyVideo?: (input: {
         audio: string;
@@ -200,6 +220,8 @@ export declare class AiLabel {
     private filming;
     /** Songs whose video failed since this server started. */
     private videoFailed;
+    /** Songs whose words were already sent to the aligner since this server started. */
+    private alignTried;
     /** Old videos already tried as storyboards since this server started. */
     private upgradeTried;
     /** The day's album is tried at most every 30 minutes, so a failing model is not hammered. */
@@ -246,6 +268,10 @@ export declare class AiLabel {
      * song's tags (seconds each), then old cover-and-waveform videos made again
      * as storyboards.
      */
+    /** The aligner this server can use: the dependency, else ElevenLabs with its key, else none. */
+    private aligner;
+    /** A song's words timed to its recording, written as 001. <title>.lrc beside it. Once per song (a failure is not retried this run). */
+    private sync;
     private filmPass;
     /**
      * One song's video: a storyboard of scenes when a model can write it and an

package/dist/ai-label.js

@@ -33,6 +33,7 @@ import { dirname, join, resolve, sep } from "node:path";
 import { aiKey } from "./ai-keys.js";
 import { composerFromEnv, OutOfCredit } from "./ai-radio.js";
 import { askJson, hasModel } from "./ai-text.js";
+import { formatLrc } from "./lyrics.js";
 import { StationError } from "./stations.js";
 /**
  * The label's first roster: invented names, one genre each, sung in English.
@@ -408,6 +409,53 @@ export async function workersAiCover(prompt, env = process.env, fetcher = fetch)
         throw new Error("the cover model answered with no picture");
     return image;
 }
+/** ElevenLabs forced alignment: the song and its words in, each word's time out. */
+export function elevenLabsAligner(key, fetcher = fetch) {
+    return async (audio, text) => {
+        const form = new FormData();
+        form.append("file", new Blob([readFileSync(audio)], { type: "audio/mpeg" }), "song.mp3");
+        form.append("text", text);
+        const response = await fetcher("https://api.elevenlabs.io/v1/forced-alignment", {
+            method: "POST",
+            headers: { "xi-api-key": key },
+            body: form,
+            signal: AbortSignal.timeout(5 * 60_000),
+        });
+        if (response.status === 401 || response.status === 402)
+            throw new OutOfCredit(`elevenlabs answered HTTP ${response.status} to the alignment`);
+        if (!response.ok)
+            throw new Error(`the alignment answered HTTP ${response.status}`);
+        const body = await response.json();
+        return (body.words ?? [])
+            .map((one) => ({ text: String(one.text ?? "").trim(), start: Number(one.start), end: Number(one.end) }))
+            .filter((one) => one.text && Number.isFinite(one.start) && Number.isFinite(one.end));
+    };
+}
+/** The sung lines of a song's words: section tags ([Chorus]) and blank lines are not sung. */
+export function sungLines(lyrics) {
+    return lyrics.split("\n").map((line) => line.trim()).filter((line) => line && !/^\[[^\]]*\]$/.test(line));
+}
+/**
+ * Timed words laid back onto the lines they were written in, in order, so
+ * each line and each word knows when it is sung.
+ */
+export function syncLines(lyrics, timed) {
+    const out = [];
+    let at = 0;
+    for (const line of sungLines(lyrics)) {
+        const words = line.split(/\s+/).filter(Boolean);
+        const placed = words.map((word) => {
+            const hit = timed[at];
+            if (hit)
+                at += 1;
+            return hit ? { text: word, start: hit.start, end: hit.end } : null;
+        }).filter((one) => one !== null);
+        if (placed.length === 0)
+            break;
+        out.push({ start: placed[0].start, end: placed[placed.length - 1].end, text: line, words: placed });
+    }
+    return out;
+}
 const STORYBOARD_SYSTEM = [
     "You storyboard music videos for original songs. Write one image prompt per scene, in order, following the song's story and mood.",
     "Keep the same main character(s), setting and color palette across scenes so it reads as one video; vary the shots (wide, close-up, action, detail).",
@@ -555,6 +603,8 @@ export class AiLabel {
     filming = null;
     /** Songs whose video failed since this server started. */
     videoFailed = new Set();
+    /** Songs whose words were already sent to the aligner since this server started. */
+    alignTried = new Set();
     /** Old videos already tried as storyboards since this server started. */
     upgradeTried = new Set();
     /** The day's album is tried at most every 30 minutes, so a failing model is not hammered. */
@@ -844,7 +894,43 @@ export class AiLabel {
      * song's tags (seconds each), then old cover-and-waveform videos made again
      * as storyboards.
      */
+    /** The aligner this server can use: the dependency, else ElevenLabs with its key, else none. */
+    aligner() {
+        if (this.deps.align !== undefined)
+            return this.deps.align;
+        const key = aiKey("ELEVENLABS_API_KEY");
+        return key ? elevenLabsAligner(key) : null;
+    }
+    /** A song's words timed to its recording, written as 001. <title>.lrc beside it. Once per song (a failure is not retried this run). */
+    async sync(artist, album, song) {
+        const align = this.aligner();
+        if (!align || !song.file || !song.lyrics || song.lrc || this.alignTried.has(song.file))
+            return false;
+        this.alignTried.add(song.file);
+        try {
+            const timed = await align(song.file, sungLines(song.lyrics).join("\n"));
+            const lines = syncLines(song.lyrics, timed);
+            if (lines.length === 0)
+                throw new Error("the aligner placed no words");
+            const file = join(album.folder, `${songBase(song)}.lrc`);
+            writeFileSync(file, formatLrc(lines, { title: song.title, artist: artist.name, album: album.title }));
+            song.lrc = file;
+            this.save();
+            return true;
+        }
+        catch (error) {
+            this.log(`could not time the words of "${song.title}": ${error.message}`);
+            return false;
+        }
+    }
     async filmPass() {
+        // Words timed first: quick, and what the station page's karaoke needs.
+        for (const artist of this.list()) {
+            for (const album of artist.albums) {
+                for (const song of album.songs)
+                    await this.sync(artist, album, song);
+            }
+        }
         const songs = this.list().flatMap((artist) => artist.albums
             .filter((album) => album.cover || album.coverTried)
             .flatMap((album) => album.songs.filter((song) => song.file && !this.videoFailed.has(song.file)).map((song) => ({ artist, album, song }))));
@@ -1184,7 +1270,8 @@ export class AiLabel {
             this.save();
             this.writeAlbumJson(artist, album);
             await this.place(artist.genre);
-            // Its video is made beside it, without holding up the next song.
+            // Its words timed to the recording, then its video, without holding up the next song.
+            await this.sync(artist, album, song).catch(() => undefined);
             void this.videos();
         }
         catch (error) {
@@ -1364,6 +1451,8 @@ export class AiLabel {
             for (const song of songs) {
                 entries.push({ name: `${folder}/${songBase(song)}.mp3`, path: song.file });
                 entries.push({ name: `${folder}/${songBase(song)}.lyrics.txt`, bytes: Buffer.from(`${song.lyrics}\n`) });
+                if (song.lrc)
+                    entries.push({ name: `${folder}/${songBase(song)}.lrc`, path: song.lrc });
                 if (song.video)
                     entries.push({ name: `${folder}/${songBase(song)}.mp4`, path: song.video });
             }

package/dist/follow-page.d.ts

@@ -0,0 +1,23 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+export interface FollowPageContext {
+    /** The signed-in account, or null. */
+    account: () => Promise<{
+        id: string;
+    } | null>;
+    follows: {
+        follow: (follower: string, streamer: string) => Promise<boolean>;
+        unfollow: (follower: string, streamer: string) => Promise<void>;
+        isFollowing: (follower: string, streamer: string) => Promise<boolean>;
+        followerCount: (streamer: string) => Promise<number>;
+    };
+    readBody: (request: IncomingMessage) => Promise<string>;
+}
+export declare const FOLLOW_CSP: string;
+/** What the link says, cleaned: an artist id, a name to show, and an https page to go back to. */
+export declare function followTarget(input: URLSearchParams): {
+    id: string;
+    name: string;
+    back: string;
+} | null;
+/** Answers /follow; false for any other path. */
+export declare function handleFollowPage(request: IncomingMessage, response: ServerResponse, path: string, url: URL, ctx: FollowPageContext): Promise<boolean>;

package/dist/follow-page.js

@@ -0,0 +1,111 @@
+/**
+ * /follow on nixamp.com: follow an artist from a /radio page (radio-pages.ts).
+ *
+ * The /radio pages are on another site (live.streaming.pit.world), where a
+ * nixamp.com session cookie is a third-party cookie and no browser sends it.
+ * So Follow there is a link here, where the session is first-party: this page
+ * says who you are following, and a same-site form (SameSite=Lax keeps a
+ * cross-site POST from carrying the session) follows or unfollows. The follow
+ * is the same follows row as following a streamer, keyed "artist:<name>".
+ *
+ *   GET  /follow?id=artist:<name>&name=<shown>&back=<https page>
+ *   POST /follow   id, name, back, act=follow|unfollow  -> 303 back to the GET
+ */
+import { createHash } from "node:crypto";
+const ID = /^artist:[a-z0-9-]{1,90}$/;
+function escape(text) {
+    return String(text).replace(/[&<>"']/g, (one) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" })[one]);
+}
+const STYLE = `
+:root{--bg:#080c09;--fg:#e8efe9;--muted:#9aa79d;--card:#0e1410;--line:#1f2a22;--accent:#3ddc84}
+*{box-sizing:border-box}html,body{margin:0;background:var(--bg);color:var(--fg);font:16px/1.5 system-ui,-apple-system,"Segoe UI",sans-serif}
+main{max-width:520px;margin:0 auto;padding:40px 16px}
+.card{background:var(--card);border:1px solid var(--line);border-radius:16px;padding:24px}
+h1{margin:0 0 4px;font-size:1.5rem;overflow-wrap:anywhere}.by{color:var(--muted);margin:0 0 16px}
+button,a.go{display:block;text-align:center;text-decoration:none;margin-top:18px;width:100%;padding:12px;border:0;border-radius:999px;background:var(--accent);color:#06120a;font:inherit;font-weight:800;cursor:pointer}
+button.quiet{background:transparent;color:var(--fg);border:1px solid var(--line)}
+a{color:var(--accent)}a:focus-visible,button:focus-visible{outline:3px solid var(--accent);outline-offset:2px}
+small{display:block;margin-top:18px;color:var(--muted)}
+`;
+const hash = (text) => `'sha256-${createHash("sha256").update(text).digest("base64")}'`;
+export const FOLLOW_CSP = ["default-src 'none'", `style-src ${hash(STYLE)}`, "img-src 'self'", "form-action 'self'", "base-uri 'none'", "frame-ancestors 'none'"].join("; ");
+function page(title, body) {
+    return `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
+<title>${escape(title)} · nixamp</title><meta name="robots" content="noindex"><link rel="icon" href="/favicon.png" type="image/png"><style>${STYLE}</style></head>
+<body><main><div class="card">${body}</div></main></body></html>`;
+}
+function send(response, status, html, method) {
+    response.writeHead(status, { "content-type": "text/html; charset=utf-8", "content-security-policy": FOLLOW_CSP, "cache-control": "no-store", "referrer-policy": "no-referrer", "x-content-type-options": "nosniff" });
+    response.end(method === "HEAD" ? undefined : html);
+}
+/** What the link says, cleaned: an artist id, a name to show, and an https page to go back to. */
+export function followTarget(input) {
+    const id = (input.get("id") ?? "").trim();
+    if (!ID.test(id))
+        return null;
+    const name = (input.get("name") ?? "").replace(/[\u0000-\u001f]/g, " ").trim().slice(0, 120) || id.slice("artist:".length);
+    let back = "";
+    try {
+        const url = new URL(input.get("back") ?? "");
+        if (url.protocol === "https:" && !url.username && !url.password)
+            back = url.href;
+    }
+    catch {
+        back = "";
+    }
+    return { id, name, back };
+}
+/** Answers /follow; false for any other path. */
+export async function handleFollowPage(request, response, path, url, ctx) {
+    if (path !== "/follow")
+        return false;
+    const method = request.method ?? "GET";
+    if (method === "POST") {
+        // Same site only: a form on another site gets no session (SameSite=Lax), and is told so here too.
+        const origin = String(request.headers.origin ?? "");
+        const host = String(request.headers["x-forwarded-host"] ?? request.headers.host ?? "").split(",")[0].trim();
+        if (origin && origin !== "null" && new URL(origin).host !== host) {
+            send(response, 403, page("Not from here", `<h1>Not from here</h1><p class="by">Follow from nixamp.com's own page.</p>`), method);
+            return true;
+        }
+        const form = new URLSearchParams(await ctx.readBody(request));
+        const target = followTarget(form);
+        const me = await ctx.account().catch(() => null);
+        if (target && me) {
+            if (form.get("act") === "unfollow")
+                await ctx.follows.unfollow(me.id, target.id);
+            else
+                await ctx.follows.follow(me.id, target.id);
+        }
+        const again = target ? `/follow?${new URLSearchParams({ id: target.id, name: target.name, ...(target.back ? { back: target.back } : {}) }).toString()}` : "/follow";
+        response.writeHead(303, { location: again, "cache-control": "no-store" });
+        response.end();
+        return true;
+    }
+    if (method !== "GET" && method !== "HEAD") {
+        response.writeHead(405, { allow: "GET, POST" }).end();
+        return true;
+    }
+    const target = followTarget(url.searchParams);
+    if (!target) {
+        send(response, 404, page("Nobody to follow", `<h1>Nobody to follow</h1><p class="by">That link does not name an artist.</p>`), method);
+        return true;
+    }
+    const back = target.back ? `<small><a href="${escape(target.back)}">← Back to ${escape(target.name)}</a></small>` : "";
+    const me = await ctx.account().catch(() => null);
+    if (!me) {
+        send(response, 200, page(`Follow ${target.name}`, `<h1>Follow ${escape(target.name)}</h1>
+<p class="by">Sign in to nixamp.com to follow the artists you hear on the radio.</p>
+<a class="go" href="/radio#music">Sign in to nixamp.com</a>
+<small>Signed in, open this page again (or press Follow on the artist's page) to follow.</small>${back}`), method);
+        return true;
+    }
+    const [following, count] = await Promise.all([ctx.follows.isFollowing(me.id, target.id), ctx.follows.followerCount(target.id).catch(() => 0)]);
+    const hidden = `<input type="hidden" name="id" value="${escape(target.id)}"><input type="hidden" name="name" value="${escape(target.name)}">${target.back ? `<input type="hidden" name="back" value="${escape(target.back)}">` : ""}`;
+    send(response, 200, page(`Follow ${target.name}`, `<h1>${escape(target.name)}</h1>
+<p class="by">${following ? "You follow them." : "Follow them to keep them with your follows on nixamp.com."} ${count} follower${count === 1 ? "" : "s"}.</p>
+<form method="post" action="/follow">${hidden}<input type="hidden" name="act" value="${following ? "unfollow" : "follow"}">
+<button type="submit"${following ? ' class="quiet"' : ""}>${following ? "Unfollow" : `Follow ${escape(target.name)}`}</button></form>
+${back}`), method);
+    return true;
+}

package/dist/lyrics.d.ts

@@ -1,18 +1,3 @@
-/**
- * The words of the song on the air, for a station's page.
- *
- * Two places they come from, in this order:
- *  1. the file itself: an ID3 USLT frame (ffprobe calls it `lyrics-eng`, or
- *     `lyrics-<description>-<lang>`), a Vorbis/APE `LYRICS` or
- *     `UNSYNCEDLYRICS` tag. Suno writes the words it sang into USLT, so our
- *     own catalog carries them already;
- *  2. nichedb.dev's lyrics, asked by artist and title, which is where a file
- *     without the tag gets them (and where ours are published back to:
- *     lyrics-publish.ts).
- * Each answer is kept: the tags with the file (by size and time, TagReader),
- * nichedb's for ten minutes, misses included, so a station of tracks nobody
- * has words for asks once per track per ten minutes and never waits long.
- */
 export interface Lyrics {
     title: string;
     artist: string;
@@ -21,7 +6,32 @@ export interface Lyrics {
     source: string;
     /** ISO 639-1 where we know the mapping, else what the tag said; "" when unsaid. */
     language: string;
+    /** When a .lrc beside the file says when each line (and word) is sung: seconds from the file's start. */
+    synced?: SyncedLine[];
+}
+export interface SyncedLine {
+    start: number;
+    end: number;
+    text: string;
+    words: Array<{
+        start: number;
+        end: number;
+        text: string;
+    }>;
 }
+/**
+ * Enhanced LRC: `[mm:ss.xx]` before each sung line, and `<mm:ss.xx>` before
+ * each word when the words are timed. Tags like `[ar:...]` and lines without a
+ * time are skipped. Ends are the next start (the last line runs 4 s).
+ */
+export declare function parseLrc(text: string): SyncedLine[];
+export declare function formatLrc(lines: SyncedLine[], head?: {
+    title?: string;
+    artist?: string;
+    album?: string;
+}): string;
+/** The .lrc that sits beside a file (same name, .lrc), or "". */
+export declare function lrcBeside(file: string): string;
 export declare const NICHEDB_LYRICS = "https://nichedb.dev";
 /** Text as it may be shown: newlines kept, control characters gone, trimmed, capped. */
 export declare function cleanLyrics(text: string): string;

package/dist/lyrics.js

@@ -13,6 +13,67 @@
  * nichedb's for ten minutes, misses included, so a station of tracks nobody
  * has words for asks once per track per ten minutes and never waits long.
  */
+import { readFileSync } from "node:fs";
+const STAMP = /^(\d{1,3}):(\d{2})(?:[.:](\d{1,3}))?$/;
+function seconds(stamp) {
+    const match = STAMP.exec(stamp);
+    if (!match)
+        return Number.NaN;
+    const fraction = match[3] ? Number(`0.${match[3]}`) : 0;
+    return Number(match[1]) * 60 + Number(match[2]) + fraction;
+}
+function stamp(at) {
+    const whole = Math.max(0, at);
+    const minutes = Math.floor(whole / 60);
+    const rest = whole - minutes * 60;
+    return `${String(minutes).padStart(2, "0")}:${rest.toFixed(2).padStart(5, "0")}`;
+}
+/**
+ * Enhanced LRC: `[mm:ss.xx]` before each sung line, and `<mm:ss.xx>` before
+ * each word when the words are timed. Tags like `[ar:...]` and lines without a
+ * time are skipped. Ends are the next start (the last line runs 4 s).
+ */
+export function parseLrc(text) {
+    const lines = [];
+    for (const raw of text.replace(/\r\n?/g, "\n").split("\n")) {
+        const head = /^\[([^\]]+)\](.*)$/.exec(raw.trim());
+        if (!head)
+            continue;
+        const start = seconds(head[1]);
+        if (!Number.isFinite(start))
+            continue;
+        const body = head[2];
+        const words = [];
+        const parts = body.split(/<([^>]+)>/);
+        if (parts.length > 1) {
+            for (let i = 1; i < parts.length; i += 2) {
+                const at = seconds(parts[i]);
+                const word = (parts[i + 1] ?? "").trim();
+                if (Number.isFinite(at) && word)
+                    words.push({ start: at, end: at, text: word });
+            }
+        }
+        const plain = (parts.length > 1 ? words.map((one) => one.text).join(" ") : body).trim();
+        if (!plain)
+            continue;
+        lines.push({ start, end: start, text: plain, words });
+    }
+    lines.sort((a, b) => a.start - b.start);
+    lines.forEach((line, i) => {
+        line.end = lines[i + 1]?.start ?? line.start + 4;
+        line.words.forEach((word, j) => { word.end = line.words[j + 1]?.start ?? line.end; });
+    });
+    return lines;
+}
+export function formatLrc(lines, head = {}) {
+    const tags = [head.artist ? `[ar:${head.artist}]` : "", head.title ? `[ti:${head.title}]` : "", head.album ? `[al:${head.album}]` : ""].filter(Boolean);
+    const body = lines.map((line) => `[${stamp(line.start)}]${line.words.length ? line.words.map((word) => ` <${stamp(word.start)}>${word.text}`).join("") : line.text}`);
+    return `${[...tags, ...body].join("\n")}\n`;
+}
+/** The .lrc that sits beside a file (same name, .lrc), or "". */
+export function lrcBeside(file) {
+    return file.replace(/\.[^./]+$/, "") + ".lrc";
+}
 export const NICHEDB_LYRICS = "https://nichedb.dev";
 const MAX_TEXT = 20_000;
 /** ID3 / ISO 639-2 codes for the languages we are likely to meet, as 639-1. */
@@ -94,8 +155,18 @@ export class LyricsFinder {
         const title = tags["title"] || known?.title || "";
         const artist = tags["artist"] || tags["album_artist"] || known?.artist || "";
         const own = lyricsFromTags(tags);
+        // Timed words beside the file (karaoke): the page follows along with them.
+        let synced = [];
+        try {
+            synced = parseLrc(readFileSync(lrcBeside(file), "utf8"));
+        }
+        catch {
+            synced = [];
+        }
         if (own)
-            return { title, artist, text: own.text, source: "embedded", language: own.language };
+            return { title, artist, text: own.text, source: "embedded", language: own.language, ...(synced.length ? { synced } : {}) };
+        if (synced.length)
+            return { title, artist, text: synced.map((line) => line.text).join("\n"), source: "embedded", language: "", synced };
         if (!title || !artist || !this.base)
             return null;
         return this.lookup(artist, title);