@audiofool/moondog 0.1.0 → 0.2.0-beta.2

package/src/integrations/apple-music/terminal-preview-player.mjs

@@ -0,0 +1,221 @@
+// Plays the agent's Apple Music previews from the terminal. Each 30-second preview is
+// downloaded to a temporary file and played by the system's own player, one after another.
+import { spawn } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { access, constants, rm, stat, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+
+// Players that play one file without a window and exit at its end, in order of preference.
+// None of them may read or write the terminal: the listening room owns it.
+const systemPlayers = {
+  darwin: [{ command: "afplay", args: [] }],
+  linux: [
+    { command: "ffplay", args: ["-nodisp", "-autoexit", "-loglevel", "quiet"] },
+    { command: "mpv", args: ["--no-video", "--no-terminal"] },
+  ],
+};
+const longestQueue = 30;
+const largestPreview = 10 * 1024 * 1024;
+// Some routes to Apple's audio host take over 20 seconds for one preview.
+const downloadTimeoutMs = 60_000;
+
+// The same addresses the website's player accepts.
+const https = (value, allowed) => {
+  try {
+    const url = new URL(value);
+    return url.protocol === "https:" && allowed(url.hostname) ? url.href : null;
+  } catch {
+    return null;
+  }
+};
+const previewHost = (host) => host === "audio-ssl.itunes.apple.com";
+const linkHost = (host) => host === "music.apple.com";
+
+/** The first system player on PATH that can play a preview, or null. */
+export async function findSystemPlayer({ platform = process.platform, environment = process.env } = {}) {
+  const folders = String(environment.PATH ?? "").split(path.delimiter).filter((folder) => path.isAbsolute(folder));
+  for (const { command, args } of systemPlayers[platform] ?? []) {
+    for (const folder of folders) {
+      const file = path.join(folder, command);
+      try {
+        if (!(await stat(file)).isFile()) continue;
+        await access(file, constants.X_OK);
+        return { command: file, args };
+      } catch {}
+    }
+  }
+  return null;
+}
+
+/**
+ * The listening room's preview player, or null when this system has no player.
+ * play(tracks, { album }) replaces the queue. subscribe(listener) reports a new queue,
+ * a song still loading, a song that could not load, each song as it starts, and the
+ * end of playback.
+ */
+export async function createTerminalPreviewPlayer({
+  platform,
+  environment,
+  fetchImpl = globalThis.fetch,
+  temporaryRoot = tmpdir(),
+} = {}) {
+  const player = await findSystemPlayer({ platform, environment });
+  if (!player) return null;
+  const listeners = new Set();
+  let folder = null;
+  let files = 0;
+  let queue = null;
+  let child = null;
+  let closed = false;
+
+  const emit = (event) => {
+    for (const listener of listeners) {
+      try { listener(event); } catch {}
+    }
+  };
+  const removeFolder = () => {
+    if (folder) rmSync(folder, { recursive: true, force: true });
+    folder = null;
+  };
+  // A process that ends without closing the player still stops the sound and leaves no files.
+  const onExit = () => {
+    child?.kill();
+    removeFolder();
+  };
+  process.once("exit", onExit);
+
+  const download = async (url, signal) => {
+    signal.throwIfAborted();
+    folder ??= mkdtempSync(path.join(temporaryRoot, "moondog-previews-"));
+    const file = path.join(folder, `${(files += 1)}.m4a`);
+    const response = await fetchImpl(url, { signal: AbortSignal.any([signal, AbortSignal.timeout(downloadTimeoutMs)]) });
+    if (!response.ok || !response.body) throw new Error(`Apple Music answered ${response.status}`);
+    const chunks = [];
+    let size = 0;
+    for await (const chunk of response.body) {
+      if ((size += chunk.byteLength) > largestPreview) throw new Error("The preview is too large");
+      chunks.push(chunk);
+    }
+    signal.throwIfAborted();
+    await writeFile(file, Buffer.concat(chunks));
+    return file;
+  };
+
+  const playFile = (file) => new Promise((resolve) => {
+    const playing = spawn(player.command, [...player.args, file], { stdio: "ignore" });
+    child = playing;
+    const finish = () => {
+      if (child === playing) child = null;
+      resolve();
+    };
+    playing.once("error", finish);
+    playing.once("exit", finish);
+  });
+
+  const start = (tracks) => {
+    const controller = new AbortController();
+    const downloads = new Map();
+    // skip() ends the song playing, or the download of the song about to play.
+    const current = { controller, skipDownload: null };
+    queue = current;
+    const fetchFile = (index) => {
+      if (!downloads.has(index)) {
+        const own = new AbortController();
+        const pending = { ready: false, abort: () => own.abort(), skipped: () => own.signal.aborted };
+        pending.file = download(tracks[index].preview, AbortSignal.any([controller.signal, own.signal]))
+          .catch(() => null)
+          .then((file) => { pending.ready = true; return file; });
+        downloads.set(index, pending);
+      }
+      return downloads.get(index);
+    };
+    void (async () => {
+      try {
+        for (let index = 0; index < tracks.length; index += 1) {
+          const { title, artist_credit, catalog_url } = tracks[index];
+          const song = { track: { title, artist_credit, catalog_url }, position: index + 1, count: tracks.length };
+          const pending = fetchFile(index);
+          if (!pending.ready) emit({ type: "loading", ...song });
+          current.skipDownload = pending.abort;
+          const file = await pending.file;
+          current.skipDownload = null;
+          if (controller.signal.aborted) break;
+          // The next preview downloads while this one plays.
+          if (index + 1 < tracks.length) void fetchFile(index + 1);
+          if (!file) {
+            // An error, a timeout, or an oversized file; a listener's /skip is not a failure.
+            if (!pending.skipped()) emit({ type: "failed", ...song });
+            continue;
+          }
+          emit({ type: "playing", ...song });
+          await playFile(file);
+          await rm(file, { force: true });
+          if (controller.signal.aborted) break;
+        }
+      } finally {
+        await Promise.all([...downloads.values()].map(async (pending) => {
+          const file = await pending.file;
+          if (file) await rm(file, { force: true });
+        }));
+        if (queue === current) {
+          queue = null;
+          emit({ type: "stopped" });
+        }
+      }
+    })();
+  };
+
+  const stop = () => {
+    if (!queue) return false;
+    queue.controller.abort();
+    queue = null;
+    child?.kill();
+    child = null;
+    emit({ type: "stopped" });
+    return true;
+  };
+
+  return {
+    // A new set of previews replaces the queue and starts from its first song.
+    play(tracks, { album } = {}) {
+      if (closed) return;
+      const accepted = (Array.isArray(tracks) ? tracks : []).map((track) => ({
+        title: String(track?.title ?? "").trim(),
+        artist_credit: String(track?.artist_credit ?? "").trim(),
+        catalog_url: https(track?.catalog_url, linkHost),
+        preview: https(track?.preview_url, previewHost),
+      })).filter((track) => track.title && track.catalog_url && track.preview).slice(0, longestQueue);
+      if (!accepted.length) return;
+      stop();
+      emit({
+        type: "queue",
+        tracks: accepted.map(({ title, artist_credit, catalog_url }) => ({ title, artist_credit, catalog_url })),
+        album: album?.title
+          ? { title: String(album.title), artist_credit: String(album.artist_credit ?? ""), catalog_url: https(album.catalog_url, linkHost) }
+          : null,
+      });
+      start(accepted);
+    },
+    // Ends the song now playing, or still loading; the next one starts after it.
+    skip() {
+      if (!queue) return false;
+      if (child) child.kill();
+      else queue.skipDownload?.();
+      return true;
+    },
+    stop,
+    subscribe(listener) {
+      listeners.add(listener);
+      return () => listeners.delete(listener);
+    },
+    close() {
+      if (closed) return;
+      closed = true;
+      stop();
+      listeners.clear();
+      process.removeListener("exit", onExit);
+      removeFolder();
+    },
+  };
+}

package/src/integrations/lastfm/history.mjs

@@ -0,0 +1,221 @@
+import { setTimeout as delay } from "node:timers/promises";
+
+import { musicHash, musicText, prepareMusicBundle } from "../../profile/music-import-bundle.mjs";
+import { readLastfmConfiguration } from "./settings.mjs";
+
+// A listener's scrobbles (every play Last.fm counted) and loved tracks, read by username.
+// Last.fm lists scrobbles newest first; the read stops at the moment it started, so new
+// scrobbles can't shift the pages underneath it.
+export const LASTFM_LIMITS = Object.freeze({
+  maximumScrobbles: 500_000,
+  maximumLovedTracks: 20_000,
+  pageSize: 200,
+  // Last.fm can suspend a key that keeps making several calls a second, so request starts
+  // are paced across all readers. A full page can take Last.fm 2 to 17 seconds to answer;
+  // three readers hide part of that wait, and more were not faster in practice.
+  requestIntervalMs: 250,
+  concurrency: 3,
+  attempts: 5,
+});
+
+export class LastfmError extends Error {
+  constructor(message, code) {
+    super(message);
+    this.name = "LastfmError";
+    this.code = code;
+  }
+}
+
+const fail = (message, code) => { throw new LastfmError(message, code); };
+
+/** A Last.fm username, or the profile link it appears in (last.fm/user/<name>). */
+export function lastfmUsername(input) {
+  let value = String(input ?? "").trim();
+  const link = value.match(/^(?:https?:\/\/)?(?:www\.)?last\.fm\/(?:[a-z]{2}\/)?user\/([^/?#\s]+)/iu);
+  if (link) {
+    try { value = decodeURIComponent(link[1]); } catch { value = ""; }
+  }
+  value = value.replace(/^@/u, "");
+  if (!/^[^\s/\\?#\u0000-\u001f\u007f]{1,64}$/u.test(value)) {
+    fail("Type your Last.fm username, as in your profile link: last.fm/user/<username>.", "lastfm_username_invalid");
+  }
+  return value;
+}
+
+const list = (value) => (Array.isArray(value) ? value : value && typeof value === "object" ? [value] : []);
+// Last.fm writes names as plain strings, as {"#text"}, or as {"name"} depending on the method.
+const name = (value) => musicText(typeof value === "string" ? value : value?.["#text"] ?? value?.name ?? "");
+const fold = (value) => value.normalize("NFKC").toLocaleLowerCase("und");
+// Last.fm identifies a track by its artist and title; the album changes from scrobble to scrobble.
+const trackKey = (artist, title) => musicHash(JSON.stringify(["lastfm-track-v1", fold(artist), fold(title)]));
+const retryable = new Set([8, 11, 16, 29]);
+// The list each paged method must answer with.
+const listFor = { "user.getrecenttracks": "recenttracks", "user.getlovedtracks": "lovedtracks" };
+
+/** Spaces request starts at least `interval` apart, whichever reader asks. */
+function requestPacer(interval, sleep, now, signal) {
+  let next = 0;
+  return async () => {
+    const at = now();
+    const wait = Math.max(0, next - at);
+    next = Math.max(at, next) + interval;
+    if (wait) await sleep(wait, undefined, { signal });
+  };
+}
+
+async function lastfmCall({ root, apiKey, method, params, fetchImpl, signal, sleep, pace, attempts }) {
+  const url = new URL(root);
+  url.search = new URLSearchParams({ method, ...params, api_key: apiKey, format: "json" }).toString();
+  for (let attempt = 1; ; attempt += 1) {
+    signal?.throwIfAborted();
+    await pace();
+    const timeout = AbortSignal.timeout(20_000);
+    let response;
+    let body = null;
+    try {
+      response = await fetchImpl(url, {
+        signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
+        headers: { "User-Agent": "Moondog (+https://github.com/Audiofool934/moondog)" },
+      });
+      body = await response.json().catch(() => null);
+    } catch (error) {
+      signal?.throwIfAborted();
+      if (attempt < attempts) { await sleep(500 * 2 ** attempt, undefined, { signal }); continue; }
+      fail("Could not reach Last.fm. Check your connection and try again.", "lastfm_unreachable");
+    }
+    const code = Number(body?.error) || 0;
+    const list = listFor[method];
+    if (response.ok && body && !code && (!list || (body[list] && typeof body[list] === "object"))) return body;
+    // Last.fm sometimes answers 200 with a page that isn't its JSON; that is retried like a busy answer,
+    // so a broken first page can't pass for an empty history.
+    const unreadable = response.ok && !code;
+    if (unreadable || retryable.has(code) || response.status === 429 || response.status >= 500) {
+      if (attempt < attempts) { await sleep((code === 29 || response.status === 429 ? 2_000 : 500) * 2 ** attempt, undefined, { signal }); continue; }
+      if (code === 29 || response.status === 429) fail("Last.fm asked Moondog to slow down. Wait a minute, then try again.", "lastfm_rate_limited");
+      fail("Last.fm is having trouble right now. Try again in a little while.", "lastfm_unavailable");
+    }
+    if (code === 6) fail("Last.fm doesn't know that username. Check it against your profile link.", "lastfm_user_not_found");
+    if (code === 17) fail("This profile hides its recent listening. In Last.fm Settings > Privacy, turn off \"Hide recent listening information\", then try again.", "lastfm_user_private");
+    if (code === 10 || code === 26) fail("Last.fm didn't accept the API key. Set up Last.fm access again.", "lastfm_key_invalid");
+    fail(`Last.fm couldn't answer (${code ? `error ${code}` : `HTTP ${response.status}`}). Try again later.`, "lastfm_failed");
+  }
+}
+
+/**
+ * Pages 2..pages of a paged method, a few at a time; page 1 is already in hand.
+ * The first page that fails stops the others, and the error waits until they have settled.
+ */
+async function remainingPages(pages, read, concurrency, stop) {
+  let next = 2;
+  let failure;
+  const worker = async () => {
+    while (next <= pages && !failure) {
+      try {
+        await read(next++);
+      } catch (error) {
+        failure ??= error;
+        stop.abort(failure);
+      }
+    }
+  };
+  await Promise.all(Array.from({ length: Math.max(0, Math.min(concurrency, pages - 1)) }, worker));
+  if (failure) throw failure;
+}
+
+export async function readLastfmHistory({
+  username,
+  subjectId,
+  capturedAt = new Date().toISOString(),
+  configuration,
+  environment = process.env,
+  fetchImpl = fetch,
+  signal,
+  onProgress = () => {},
+  sleep = delay,
+  now = Date.now,
+  limits = LASTFM_LIMITS,
+} = {}) {
+  const user = lastfmUsername(username);
+  const access = configuration ?? await readLastfmConfiguration(environment);
+  if (!access?.apiKey) fail("Moondog needs a Last.fm API key before it can read your history.", "lastfm_key_missing");
+  // One read shares one pace and one stop, so a failed page also stops its siblings.
+  const stop = new AbortController();
+  const readSignal = signal ? AbortSignal.any([signal, stop.signal]) : stop.signal;
+  const pace = requestPacer(limits.requestIntervalMs, sleep, now, readSignal);
+  const call = (method, params) => lastfmCall({
+    root: access.root, apiKey: access.apiKey, method, params: { user, ...params },
+    fetchImpl, signal: readSignal, sleep, pace, attempts: limits.attempts,
+  });
+  const until = Math.floor(Date.parse(capturedAt) / 1_000);
+  const pageSize = limits.pageSize;
+
+  const scrobbles = [];
+  const takeScrobbles = (body) => {
+    for (const track of list(body?.recenttracks?.track)) {
+      const at = Number(track?.date?.uts);
+      // The track playing right now has no date and isn't a scrobble yet.
+      if (track?.["@attr"]?.nowplaying === "true" || !Number.isSafeInteger(at) || at <= 0 || at > until) continue;
+      const title = name(track.name);
+      const artist = name(track.artist);
+      if (title && artist) scrobbles.push({ at, title, artist, album: name(track.album) });
+    }
+  };
+  const first = await call("user.getrecenttracks", { limit: pageSize, page: 1, to: until });
+  const attributes = first?.recenttracks?.["@attr"] ?? {};
+  const total = Number(attributes.total) || 0;
+  const pages = Math.min(Number(attributes.totalPages) || 1, Math.ceil(limits.maximumScrobbles / pageSize));
+  const expected = Math.min(total, limits.maximumScrobbles);
+  takeScrobbles(first);
+  onProgress({ stage: "scrobbles", count: scrobbles.length, total: expected });
+  await remainingPages(pages, async (page) => {
+    takeScrobbles(await call("user.getrecenttracks", { limit: pageSize, page, to: until }));
+    onProgress({ stage: "scrobbles", count: scrobbles.length, total: expected });
+  }, limits.concurrency, stop);
+
+  const loved = [];
+  const takeLoved = (body) => {
+    for (const track of list(body?.lovedtracks?.track)) {
+      const title = name(track?.name);
+      const artist = name(track?.artist);
+      if (title && artist) loved.push({ title, artist });
+    }
+  };
+  const lovedFirst = await call("user.getlovedtracks", { limit: pageSize, page: 1 });
+  const lovedPages = Math.min(Number(lovedFirst?.lovedtracks?.["@attr"]?.totalPages) || 1, Math.ceil(limits.maximumLovedTracks / pageSize));
+  takeLoved(lovedFirst);
+  onProgress({ stage: "loved", count: loved.length });
+  await remainingPages(lovedPages, async (page) => {
+    takeLoved(await call("user.getlovedtracks", { limit: pageSize, page }));
+    onProgress({ stage: "loved", count: loved.length });
+  }, limits.concurrency, stop);
+
+  if (!scrobbles.length && !loved.length) {
+    fail("This Last.fm profile has no scrobbles or loved tracks yet.", "lastfm_history_empty");
+  }
+  scrobbles.sort((left, right) => left.at - right.at || trackKey(left.artist, left.title).localeCompare(trackKey(right.artist, right.title)));
+  const rows = [
+    ...scrobbles.map((row) => ({
+      id: trackKey(row.artist, row.title), title: row.title, artist: row.artist, album: row.album,
+      occurredAt: new Date(row.at * 1_000).toISOString(), member: "lastfm-scrobbles.json",
+    })),
+    ...loved.map((row) => ({ id: trackKey(row.artist, row.title), title: row.title, artist: row.artist, member: "lastfm-loved-tracks.json" })),
+  ];
+  // The fingerprint covers exactly what was read, so the same history imports once.
+  const bytes = Buffer.from(JSON.stringify({ user, until, scrobbles, loved }));
+  const truncated = total > scrobbles.length && pages * pageSize >= limits.maximumScrobbles;
+  const prepared = prepareMusicBundle({
+    provider: "lastfm", subjectId, capturedAt, rows,
+    digest: musicHash(bytes), size: bytes.length,
+    members: ["lastfm-scrobbles.json", ...(loved.length ? ["lastfm-loved-tracks.json"] : [])],
+    fileName: `last.fm/user/${user}`,
+    scopeNote: truncated
+      ? `Your latest ${limits.maximumScrobbles.toLocaleString("en-US")} scrobbles, the most Moondog reads at once, and your loved tracks.`
+      : "Every scrobble Last.fm counted, and your loved tracks.",
+  });
+  prepared.preview.source = "lastfm";
+  prepared.preview.lovedTracks = loved.length;
+  prepared.preview.totalScrobbles = total;
+  prepared.preview.truncated = truncated;
+  prepared.preview.profileUrl = `https://www.last.fm/user/${encodeURIComponent(user)}`;
+  return prepared;
+}

package/src/integrations/lastfm/settings.mjs

@@ -0,0 +1,75 @@
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+
+import { resolveMoondogAuthFile } from "../../runtime/pi/persistent-credential-store.mjs";
+
+const SETTINGS_VERSION = 1;
+// Last.fm API keys are 32 hexadecimal characters. Reading public history needs only the key, never the shared secret.
+const apiKeyPattern = /^[a-f0-9]{32}$/u;
+
+export const LASTFM_API_ROOT = "https://ws.audioscrobbler.com/2.0/";
+
+// Moondog's own key, registered to the project. Reading public listening needs only this
+// key, so it ships with the source; a listener's own key in settings or the environment wins.
+export const LASTFM_PROJECT_API_KEY = "396133c344735725f65b322ece17a2da";
+
+function settingsError(message, cause) {
+  const error = new Error(message, cause ? { cause } : undefined);
+  error.name = "MoondogLastfmSettingsError";
+  error.code = "lastfm_settings_invalid";
+  return error;
+}
+
+export function normalizeLastfmApiKey(value) {
+  const key = typeof value === "string" ? value.trim().toLowerCase() : "";
+  if (!apiKeyPattern.test(key)) {
+    throw settingsError("A Last.fm API key is 32 letters and numbers. Copy the API key, not the shared secret.");
+  }
+  return key;
+}
+
+export function resolveLastfmSettingsFile(environment = process.env) {
+  return path.join(path.dirname(resolveMoondogAuthFile(environment)), "lastfm.json");
+}
+
+// Tests and local mirrors may point Moondog at another API root; only https, or http on this machine.
+function apiRoot(environment) {
+  const configured = environment.MOONDOG_LASTFM_API_ROOT?.trim();
+  if (!configured) return LASTFM_API_ROOT;
+  let url;
+  try { url = new URL(configured); } catch { throw settingsError("MOONDOG_LASTFM_API_ROOT must be a URL."); }
+  if (url.protocol !== "https:" && !(url.protocol === "http:" && ["127.0.0.1", "localhost", "[::1]"].includes(url.hostname))) {
+    throw settingsError("MOONDOG_LASTFM_API_ROOT must use https, or http on this machine.");
+  }
+  return url.href;
+}
+
+/** The key Moondog reads Last.fm with: the environment, then this machine's settings, then the project's own. */
+export async function readLastfmConfiguration(environment = process.env) {
+  const root = apiRoot(environment);
+  const environmentKey = environment.MOONDOG_LASTFM_API_KEY?.trim();
+  if (environmentKey) return { apiKey: normalizeLastfmApiKey(environmentKey), source: "environment", root };
+  let source;
+  try {
+    source = await readFile(resolveLastfmSettingsFile(environment), "utf8");
+  } catch (error) {
+    if (error?.code !== "ENOENT") throw settingsError("Moondog could not read its Last.fm settings.", error);
+  }
+  if (source !== undefined) {
+    let document;
+    try { document = JSON.parse(source); } catch (error) { throw settingsError("Moondog's Last.fm settings are not valid JSON.", error); }
+    if (!document || typeof document !== "object" || document.version !== SETTINGS_VERSION) {
+      throw settingsError("Moondog's Last.fm settings are in an unknown format.");
+    }
+    return { apiKey: normalizeLastfmApiKey(document.api_key), source: "settings", root };
+  }
+  return LASTFM_PROJECT_API_KEY ? { apiKey: LASTFM_PROJECT_API_KEY, source: "moondog", root } : undefined;
+}
+
+export async function writeLastfmConfiguration(apiKey, environment = process.env) {
+  const normalized = normalizeLastfmApiKey(apiKey);
+  const settingsFile = resolveLastfmSettingsFile(environment);
+  await mkdir(path.dirname(settingsFile), { recursive: true, mode: 0o700 });
+  await writeFile(settingsFile, `${JSON.stringify({ version: SETTINGS_VERSION, api_key: normalized }, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
+  return { apiKey: normalized, source: "settings", root: apiRoot(environment) };
+}