nixamp 0.10.2 → 0.11.0

package/dist/server.d.ts

@@ -11,6 +11,8 @@ import { SignIn } from "./oauth.ts";
 import { Owner } from "./owner.ts";
 import { Directory } from "./directory.ts";
 import { PartyLine } from "./partyline.ts";
+import { HlsPackagers } from "./hls.ts";
+import { Enricher } from "./enrich.ts";
 import { Follows } from "./follows.ts";
 import { Favorites } from "./favorites.ts";
 import { Catalogs } from "./catalogs.ts";
@@ -422,6 +424,10 @@ export interface HandlerOptions {
     ffprobe?: string[];
     /** yt-dlp, which turns a pasted page into a media address. Null when there is none. */
     ytdlp?: string[] | null;
+    /** Channels as HLS, for Safari on a phone, which plays a live stream no other way. */
+    hls?: HlsPackagers;
+    /** What a name is -- a film, a channel, a fixture -- asked of nichedb.dev and remembered. */
+    enricher?: Enricher;
     /** A Netscape cookies file for sites that want a signed-in browser, when there is one. */
     cookies?: string;
     /** Who is listening, for the admin view. */

package/dist/server.js

@@ -34,6 +34,8 @@ import { stateDir } from "./daemon.js";
 import { readSession } from "./session.js";
 import { Directory, ENDED_TTL_MS, parseAnnouncement } from "./directory.js";
 import { PartyLine, telnyxSms } from "./partyline.js";
+import { HlsPackagers, withKey } from "./hls.js";
+import { DEFAULT_SITE as NICHEDB, Enricher } from "./enrich.js";
 import { contentTypeFor, downloadArgs, fileNameFor, inputArgsFor, linkChannelId, playableLink, resolveLink, saveFormat, } from "./links.js";
 import { CALL_IN_NUMBER, OPT_IN_PATH, optInPage } from "./optin.js";
 import pg from "pg";
@@ -2316,6 +2318,35 @@ export function createHandler(engine, options) {
         // own, the way a catalog entry is played: started for whoever asked,
         // stopped a minute after the last viewer leaves. Open to anyone holding
         // the link, like picking something from a catalog.
+        // --- what is this? ------------------------------------------------------
+        //
+        // A file name, a playlist entry, a channel: the poster, the logo, the
+        // year, the rating, from nichedb.dev, remembered here so a library is
+        // asked about once. Open to whoever holds the link, like the playlist.
+        if (path === "/api/enrich" && request.method === "GET") {
+            const name = (url.searchParams.get("name") ?? "").trim().slice(0, 300);
+            if (name === "") {
+                json(response, 400, { error: "name is required" });
+                return;
+            }
+            if (!options.enricher) {
+                json(response, 200, { match: null });
+                return;
+            }
+            const kinds = ["auto", "title", "channel", "fixture"];
+            const asked = url.searchParams.get("kind") ?? "auto";
+            const kind = kinds.includes(asked) ? asked : "auto";
+            const year = Number(url.searchParams.get("year")) || null;
+            const match = await options.enricher.lookup(name, kind, year);
+            response.writeHead(200, {
+                ...CORS,
+                "content-type": "application/json; charset=utf-8",
+                // A miss is worth asking again in a few hours; a hit lasts the day.
+                "cache-control": match ? "public, max-age=3600" : "public, max-age=600",
+            });
+            response.end(JSON.stringify({ match }));
+            return;
+        }
         if (path === "/api/links/play" && request.method === "POST") {
             let body = {};
             try {
@@ -2441,8 +2472,52 @@ export function createHandler(engine, options) {
         if (path.startsWith("/api/channels/") && options.channels) {
             const channels = options.channels;
             const rest = path.slice("/api/channels/".length);
-            const [rawId, action] = rest.split("/");
+            const [rawId, action, file] = rest.split("/");
             const id = cleanId(rawId);
+            // The same channel as HLS: a playlist of short files, which is what
+            // Safari on an iPhone plays live -- it will not take the endless MP4
+            // below, and spun on it a few times before giving up. Packaged on
+            // demand, copying the fragments the browser would have got.
+            if (action === "hls" && request.method === "GET") {
+                if (!options.hls) {
+                    json(response, 503, { error: "this server cannot package HLS" });
+                    return;
+                }
+                if (!channels.has(id)) {
+                    json(response, 404, { error: "nothing is playing on that channel" });
+                    return;
+                }
+                if (file === "index.m3u8") {
+                    const playlist = await options.hls.playlist(id);
+                    if (playlist === null) {
+                        json(response, 503, { error: "that channel could not be packaged as HLS yet; try again in a moment" });
+                        return;
+                    }
+                    response.writeHead(200, {
+                        ...CORS,
+                        "content-type": "application/vnd.apple.mpegurl",
+                        "cache-control": "no-store",
+                    });
+                    // The key rides on every segment line: a browser drops the query
+                    // when it resolves a segment against the playlist.
+                    response.end(withKey(playlist, url.searchParams.get("k") ?? ""));
+                    return;
+                }
+                const segment = options.hls.segment(id, file ?? "");
+                if (segment === "") {
+                    json(response, 404, { error: "no such segment" });
+                    return;
+                }
+                watch(request, response, "stream", id);
+                response.writeHead(200, {
+                    ...CORS,
+                    "content-type": "video/mp2t",
+                    "cache-control": "no-store",
+                    "content-length": statSync(segment).size,
+                });
+                createReadStream(segment).pipe(response);
+                return;
+            }
             if (action === undefined && request.method === "GET") {
                 // Listening. The response is the fan-out target: whatever ffmpeg
                 // produces for this channel is written to it until one end goes away.
@@ -3474,6 +3549,20 @@ export async function serve(argv, version = "0.1.0") {
         onStart: (info) => console.log(`  ${info.name} is publishing to "${info.id}" (${info.format} over ${info.via}).`),
         onEnd: (info) => console.log(`  "${info.id}" stopped.`),
     });
+    // Channels as HLS, on demand, for Safari on a phone: one ffmpeg copying a
+    // channel's fragments into short files while somebody is asking for them.
+    // What things are, from nichedb.dev, remembered beside the keys so a
+    // library is asked about once across restarts.
+    const enricher = new Enricher({
+        site: process.env["NIXAMP_NICHEDB"] || NICHEDB,
+        cacheFile: join(stateDir(), "enrich.json"),
+        onEvent: (message) => console.log(message),
+    });
+    const hls = new HlsPackagers({
+        ffmpeg: tools.ffmpeg,
+        listen: (id, listener) => channels.listen(id, listener),
+        onEvent: (message) => console.log(message),
+    });
     // The channels this server was carrying when it was last stopped, put back
     // on. A server is restarted to pick up a new version, which is often, and
     // every restart used to take CNN off the air until somebody noticed.
@@ -3825,6 +3914,8 @@ export async function serve(argv, version = "0.1.0") {
         // for the sites that will not talk to a datacenter without one.
         ytdlp: tools.ytdlp ?? null,
         cookies: cookiesFile(),
+        hls,
+        enricher,
         ...(tls ? { tls } : {}),
         // Untagged, so a directory of five thousand files answers at once; the
         // tags follow through `tag` below.
@@ -4244,6 +4335,8 @@ export async function serve(argv, version = "0.1.0") {
     });
     const shutdown = () => {
         rtmp?.stop();
+        enricher.save();
+        hls.stopAll();
         channels.stopAll();
         ingest?.stopRtmp();
         ingest?.close();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nixamp",
-  "version": "0.10.2",
+  "version": "0.11.0",
   "description": "It really whips the terminal's ass. A Winamp-shaped audio player for your terminal.",
   "license": "MIT",
   "type": "module",

package/src/audio.ts

@@ -468,7 +468,11 @@ export function videoArgs(codecs: Codecs, capKbps = 0): string[] {
   const keepAudio = !transportStream && (codecs.audio === "aac" || codecs.audio === "mp3");
   return [
     "-c:v", keepVideo ? "copy" : "libx264",
-    ...(keepVideo ? [] : ["-preset", "veryfast", "-crf", "23", "-pix_fmt", "yuv420p"]),
+    // A keyframe every two seconds when encoding. A fragment starts on a
+    // keyframe, so this is how soon a joiner sees a picture -- and an HLS
+    // segment, which is cut on keyframes too, was ten seconds long on
+    // x264's default and made a phone wait thirty before it played.
+    ...(keepVideo ? [] : ["-preset", "veryfast", "-crf", "23", "-pix_fmt", "yuv420p", "-g", "48", "-keyint_min", "48", "-sc_threshold", "0"]),
     "-c:a", keepAudio ? "copy" : "aac",
     ...(keepAudio ? [] : ["-b:a", "160k", "-ac", "2"]),
     "-f", "mp4",

package/src/enrich.ts

@@ -0,0 +1,294 @@
+/**
+ * What is this, really?
+ *
+ * A file called "Top.Gun.Maverick.2022.1080p.WEB-DL.mkv" is a film with a
+ * poster, a year and a rating; a playlist entry called "US: ESPN2 HD" is a
+ * channel with a logo, a country and a category; "Lakers at Celtics" is a
+ * fixture with a score. nixamp knows none of that on its own -- ffprobe reads
+ * tags, and a torrent's tags are its file name -- so it asks nichedb.dev,
+ * which keeps the titles, channels and fixtures every profullstack site is
+ * built on, and answers a name with the best match and a score.
+ *
+ * Asked once per name and remembered: a library of five thousand files must
+ * not become five thousand requests a day, and a channel that was ESPN2
+ * yesterday is ESPN2 today. Misses are remembered too, for less long, so a
+ * file nichedb has never heard of is not asked about every time it plays.
+ */
+import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+
+/** Where the answers come from, unless a deployment says otherwise. */
+export const DEFAULT_SITE = "https://nichedb.dev";
+/** How long a hit is believed. Titles and channels change on the order of months. */
+export const HIT_TTL_MS = 7 * 24 * 60 * 60 * 1000;
+/** How long a miss is believed: nichedb's catalogue is still filling in. */
+export const MISS_TTL_MS = 6 * 60 * 60 * 1000;
+/** A fixture's score is stale in a minute. */
+export const FIXTURE_TTL_MS = 60 * 1000;
+/**
+ * Below this the best match is a guess, and a wrong poster is worse than none.
+ * Measured: "Severance" against a channel called "Sever" scored 0.45, and
+ * "Lakers at Celtics" against "Rangers at Celtic" 0.44. Half is above both.
+ */
+export const MIN_SCORE = 0.5;
+/** A weaker score is still taken when one name plainly begins with the other. */
+export const PREFIX_SCORE = 0.42;
+/** How many answers the cache keeps before the oldest go. */
+export const MAX_ENTRIES = 5000;
+
+export type EnrichKind = "auto" | "title" | "channel" | "fixture";
+
+export interface Enriched {
+  /** What nichedb says it is. */
+  kind: "title" | "channel" | "fixture";
+  title: string;
+  /** For a title, its year; for a fixture, when it starts. */
+  year: number | null;
+  /** A poster, a logo, or nothing. */
+  image: string | null;
+  summary: string | null;
+  /** nichedb's page for it, for a link. */
+  page: string;
+  score: number;
+  /** The rest, as the collection shapes it: rating, genres, country, scores… */
+  data: Record<string, unknown>;
+  tags: string[];
+}
+
+interface Cached {
+  at: number;
+  hit: Enriched | null;
+}
+
+/** nichedb's answer to /api/v1/match, as much of it as is read here. */
+interface MatchAnswer {
+  parsed?: { name?: string; year?: number | null; kind?: string; season?: number | null; episode?: number | null };
+  items?: {
+    kind?: string;
+    title?: string;
+    summary?: string | null;
+    image_url?: string | null;
+    published_at?: string | null;
+    page?: string;
+    score?: number;
+    data?: Record<string, unknown>;
+    tags?: string[];
+  }[];
+}
+
+/** The collection and kind a name is asked about, from what the caller knows. */
+export function whereToAsk(kind: EnrichKind, parsedKind?: string): { collection: string; kind: string } | null {
+  const k = kind === "auto" ? parsedKind ?? "" : kind;
+  switch (k) {
+    case "channel":
+      return { collection: "channels", kind: "channel" };
+    case "fixture":
+      return { collection: "sports", kind: "fixture" };
+    case "title":
+    case "movie":
+    case "series":
+      return { collection: "screen", kind: "title" };
+    default:
+      // Music and the rest: nichedb has no answer worth a poster yet.
+      return null;
+  }
+}
+
+/** The key one name is remembered under: case and spacing do not make it a different name. */
+export function cacheKey(name: string, kind: EnrichKind, year: number | null): string {
+  return `${kind}|${year ?? ""}|${name.trim().toLowerCase().replace(/\s+/g, " ")}`;
+}
+
+/** Whether a stored answer is still worth believing. */
+export function fresh(entry: Cached, now: number): boolean {
+  const ttl = entry.hit === null ? MISS_TTL_MS : entry.hit.kind === "fixture" ? FIXTURE_TTL_MS : HIT_TTL_MS;
+  return now - entry.at < ttl;
+}
+
+/** The best of nichedb's answers, or nothing when the best is a guess. */
+export function pickBest(answer: MatchAnswer, asked: string): Enriched | null {
+  const items = answer.items ?? [];
+  const wanted = asked.trim().toLowerCase();
+  type Item = NonNullable<MatchAnswer["items"]>[number];
+  let best: Item | undefined;
+  const isExact = (item: Item | undefined): boolean =>
+    item !== undefined && String(item.title ?? "").toLowerCase() === wanted;
+  const plain = (s: string): string => s.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
+  const askedPlain = plain(wanted);
+  for (const item of items) {
+    const exact = isExact(item);
+    const score = Number(item.score ?? 0);
+    const titlePlain = plain(String(item.title ?? ""));
+    // Whole words: "Top Gun Maverick Extended" begins with "Top Gun Maverick",
+    // but "Severance" does not begin with a channel called "Sever".
+    const prefix =
+      titlePlain.length >= 4 &&
+      askedPlain.length >= 4 &&
+      (askedPlain.startsWith(`${titlePlain} `) || titlePlain.startsWith(`${askedPlain} `));
+    if (!exact && score < (prefix ? PREFIX_SCORE : MIN_SCORE)) continue;
+    // An exact title beats any score; among the rest, the score decides.
+    if (!best) best = item;
+    else if (exact && !isExact(best)) best = item;
+    else if (!isExact(best) && score > Number(best.score ?? 0)) best = item;
+  }
+  if (!best) return null;
+  const kind = best.kind === "channel" || best.kind === "fixture" ? best.kind : "title";
+  const year = typeof best.data?.["year"] === "number"
+    ? (best.data["year"] as number)
+    : best.published_at
+      ? new Date(best.published_at).getUTCFullYear() || null
+      : null;
+  return {
+    kind,
+    title: String(best.title ?? ""),
+    year: Number.isFinite(year) ? year : null,
+    image: best.image_url ?? null,
+    summary: best.summary ?? null,
+    page: String(best.page ?? ""),
+    score: Number(best.score ?? 0),
+    data: best.data ?? {},
+    tags: best.tags ?? [],
+  };
+}
+
+export interface EnricherOptions {
+  site?: string;
+  fetch?: typeof globalThis.fetch;
+  /** Where answers are kept between runs; none means memory only. */
+  cacheFile?: string;
+  now?: () => number;
+  onEvent?: (message: string) => void;
+}
+
+export class Enricher {
+  private readonly site: string;
+  private readonly fetcher: typeof globalThis.fetch;
+  private readonly now: () => number;
+  private readonly cache = new Map<string, Cached>();
+  private readonly inflight = new Map<string, Promise<Enriched | null>>();
+  private saveTimer: ReturnType<typeof setTimeout> | null = null;
+  private dirty = false;
+
+  constructor(private readonly options: EnricherOptions = {}) {
+    this.site = (options.site ?? DEFAULT_SITE).replace(/\/+$/, "");
+    this.fetcher = options.fetch ?? globalThis.fetch;
+    this.now = options.now ?? Date.now;
+    this.load();
+  }
+
+  /** How many names are remembered. */
+  get size(): number {
+    return this.cache.size;
+  }
+
+  /**
+   * What a name is, from the cache or from nichedb.
+   *
+   * `kind` narrows the question when the caller knows: a live channel is a
+   * channel however its name reads. `auto` lets nichedb's parser decide from
+   * the name itself, which is right for a file.
+   */
+  async lookup(name: string, kind: EnrichKind = "auto", year: number | null = null): Promise<Enriched | null> {
+    const asked = String(name ?? "").trim();
+    if (asked === "") return null;
+    const key = cacheKey(asked, kind, year);
+    const had = this.cache.get(key);
+    if (had && fresh(had, this.now())) return had.hit;
+    const running = this.inflight.get(key);
+    if (running) return running;
+    const work = this.ask(asked, kind, year)
+      .then((hit) => {
+        this.remember(key, hit);
+        return hit;
+      })
+      .catch((error: unknown) => {
+        this.options.onEvent?.(`  nichedb did not answer for "${asked}": ${(error as Error).message}`);
+        // Not remembered: a network fault is not a miss.
+        return had?.hit ?? null;
+      })
+      .finally(() => this.inflight.delete(key));
+    this.inflight.set(key, work);
+    return work;
+  }
+
+  private async ask(name: string, kind: EnrichKind, year: number | null): Promise<Enriched | null> {
+    // Two round trips at most: nichedb parses the name; when the caller did
+    // not say what it is, the first answer's reading says where to look.
+    const first = new URLSearchParams({ q: name, limit: "3" });
+    if (year !== null) first.set("year", String(year));
+    const where = whereToAsk(kind);
+    if (where) {
+      first.set("collection", where.collection);
+      first.set("kind", where.kind);
+    }
+    const answer = await this.get(`/api/v1/match?${first}`);
+    if (where) return pickBest(answer, answer.parsed?.name ?? name);
+    const guessed = whereToAsk("auto", answer.parsed?.kind);
+    if (!guessed) return null;
+    const second = new URLSearchParams(first);
+    second.set("collection", guessed.collection);
+    second.set("kind", guessed.kind);
+    // The year the name carried narrows the second question.
+    if (year === null && answer.parsed?.year) second.set("year", String(answer.parsed.year));
+    return pickBest(await this.get(`/api/v1/match?${second}`), answer.parsed?.name ?? name);
+  }
+
+  private async get(path: string): Promise<MatchAnswer> {
+    const response = await this.fetcher(`${this.site}${path}`, {
+      headers: { accept: "application/json", "user-agent": "nixamp (+https://nixamp.com)" },
+      signal: AbortSignal.timeout(15_000),
+    });
+    if (!response.ok) throw new Error(`nichedb answered ${response.status}`);
+    return (await response.json()) as MatchAnswer;
+  }
+
+  private remember(key: string, hit: Enriched | null): void {
+    this.cache.set(key, { at: this.now(), hit });
+    if (this.cache.size > MAX_ENTRIES) {
+      // Oldest first: a Map remembers insertion order.
+      const drop = this.cache.size - MAX_ENTRIES;
+      let n = 0;
+      for (const k of this.cache.keys()) {
+        if (n++ >= drop) break;
+        this.cache.delete(k);
+      }
+    }
+    this.dirty = true;
+    this.scheduleSave();
+  }
+
+  private load(): void {
+    if (!this.options.cacheFile) return;
+    try {
+      const parsed = JSON.parse(readFileSync(this.options.cacheFile, "utf8")) as Record<string, Cached>;
+      for (const [k, v] of Object.entries(parsed)) {
+        if (v && typeof v.at === "number") this.cache.set(k, v);
+      }
+    } catch {
+      // No cache yet, or one that is not JSON: start empty.
+    }
+  }
+
+  private scheduleSave(): void {
+    if (!this.options.cacheFile || this.saveTimer) return;
+    this.saveTimer = setTimeout(() => {
+      this.saveTimer = null;
+      this.save();
+    }, 2000);
+    this.saveTimer.unref?.();
+  }
+
+  /** Write the cache now. Called on a timer, and by whoever is shutting down. */
+  save(): void {
+    if (!this.options.cacheFile || !this.dirty) return;
+    try {
+      mkdirSync(dirname(this.options.cacheFile), { recursive: true });
+      const tmp = `${this.options.cacheFile}.tmp`;
+      writeFileSync(tmp, JSON.stringify(Object.fromEntries(this.cache)));
+      renameSync(tmp, this.options.cacheFile);
+      this.dirty = false;
+    } catch (error) {
+      this.options.onEvent?.(`  could not save the enrichment cache: ${(error as Error).message}`);
+    }
+  }
+}