@audiofool/moondog 0.1.0 → 0.2.0-beta.3

package/src/profile/listening-history-store.mjs

@@ -1,4 +1,4 @@
-import { randomUUID } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 import { chmod, mkdir } from "node:fs/promises";
 import path from "node:path";
 
@@ -9,8 +9,23 @@ import {
   createListenerCorrectionRetraction,
 } from "./listener-corrections.mjs";
 import { projectListeningProfile } from "./listening-profile-projection.mjs";
+import {
+  listeningFamiliarity, markMoondogStartedPlays, mixSlot, suggestionEntry, suggestionIdentities, SUGGESTION_WINDOW,
+} from "./moondog-suggestions.mjs";
+import { ListenerProfileStore, listenerProfileSchema } from "./listener-profile-store.mjs";
+
+export const LISTENING_HISTORY_SCHEMA_VERSION = 7;
 
-export const LISTENING_HISTORY_SCHEMA_VERSION = 6;
+// The tables a cached projection, build input, Avoid gate or status reads, apart from the
+// suggestions ledger (listeningDataVersion). Rows are only added to the first ones: the one
+// update to a stored play, which records the play that supersedes it, is made with its row in
+// listening_event_supersessions. Imports update their batch rows, so those are read in full.
+// A new table those caches read, or a new update or delete, belongs here.
+const APPENDED_EVIDENCE_TABLES = Object.freeze([
+  "track_refs", "listening_events", "listening_event_supersessions", "listening_event_mirrors",
+  "spotify_profile_evidence", "taste_events",
+]);
+const UPDATED_EVIDENCE_TABLES = Object.freeze(["import_batches", "profile_import_batches"]);
 
 export function resolveListeningHistoryPath(environment = process.env) {
   return path.join(resolveMoondogStateDirectory(environment), "listening-history.sqlite");
@@ -48,6 +63,8 @@ const schema = `
 
   CREATE INDEX IF NOT EXISTS listening_events_subject_time
     ON listening_events(subject_id, occurred_at DESC);
+  CREATE INDEX IF NOT EXISTS listening_events_subject_track
+    ON listening_events(subject_id, track_ref_id, track_ref_revision);
 
   CREATE TABLE IF NOT EXISTS source_cursors (
     source_key TEXT PRIMARY KEY NOT NULL,
@@ -55,6 +72,18 @@ const schema = `
     updated_at TEXT NOT NULL
   ) STRICT;
 
+  -- Spotify keeps only a listener's last 50 plays. When a sync finds the window full,
+  -- plays between the last stored one and the oldest returned one may be gone for good.
+  CREATE TABLE IF NOT EXISTS listening_coverage_gaps (
+    subject_id TEXT NOT NULL,
+    source_key TEXT NOT NULL,
+    gap_start TEXT NOT NULL,
+    gap_end TEXT NOT NULL,
+    detected_at TEXT NOT NULL,
+    PRIMARY KEY (subject_id, gap_start, gap_end),
+    CHECK (gap_start < gap_end)
+  ) STRICT;
+
   CREATE TABLE IF NOT EXISTS import_batches (
     import_batch_id TEXT PRIMARY KEY NOT NULL,
     subject_id TEXT NOT NULL,
@@ -98,6 +127,18 @@ const schema = `
       REFERENCES import_batches(import_batch_id)
   ) STRICT;
 
+  -- A Last.fm scrobble that repeats a play another service recorded stays stored,
+  -- linked to that play, and is left out of every count.
+  CREATE TABLE IF NOT EXISTS listening_event_mirrors (
+    mirror_event_id TEXT PRIMARY KEY NOT NULL,
+    original_event_id TEXT NOT NULL UNIQUE,
+    created_at TEXT NOT NULL,
+    FOREIGN KEY (mirror_event_id)
+      REFERENCES listening_events(listening_event_id),
+    FOREIGN KEY (original_event_id)
+      REFERENCES listening_events(listening_event_id)
+  ) STRICT;
+
   CREATE TABLE IF NOT EXISTS profile_import_batches (
     profile_import_id TEXT PRIMARY KEY NOT NULL,
     subject_id TEXT NOT NULL,
@@ -158,6 +199,28 @@ const schema = `
   CREATE UNIQUE INDEX IF NOT EXISTS taste_events_one_retraction
     ON taste_events(retracts_taste_event_id)
     WHERE retracts_taste_event_id IS NOT NULL;
+
+  -- What Moondog itself queued or played, written from the host's accepted receipts. Plays
+  -- that match one are left out of the listener's own attention when the profile is read
+  -- (src/profile/moondog-suggestions.mjs). Provider track identities never leave the host.
+  CREATE TABLE IF NOT EXISTS moondog_suggestions (
+    suggestion_id TEXT PRIMARY KEY NOT NULL,
+    service TEXT NOT NULL,
+    action TEXT NOT NULL,
+    provider_track_id TEXT,
+    isrc TEXT,
+    track_key TEXT,
+    release_key TEXT,
+    title TEXT NOT NULL,
+    artist_credit TEXT NOT NULL,
+    requested_at TEXT NOT NULL,
+    mix_slot TEXT
+  ) STRICT;
+
+  CREATE INDEX IF NOT EXISTS moondog_suggestions_time
+    ON moondog_suggestions(requested_at);
+
+  ${listenerProfileSchema}
 `;
 
 function listenerCorrectionError(code, message) {
@@ -207,6 +270,7 @@ function normalizedBundle(value, maximumRecords = 50) {
     capturedAt: new Date(value.captured_at).toISOString(),
     trackRefs: value.track_refs,
     listeningEvents: value.listening_events,
+    windowFull: value.window_full === true,
   };
 }
 
@@ -321,7 +385,8 @@ function normalizedProfileInput(value, batch) {
       !isPlainObject(record.provenance) ||
       record.provenance.source_kind !== "import" ||
       !(record.provenance.source_system === "spotify_account_data" ||
-        (["youtube_music", "qq_music", "netease"].includes(record.provenance.source_system) &&
+        (record.provenance.source_system === "spotify_web_api" && profileImport.source_key === "spotify.web_api_library") ||
+        (["youtube_music", "qq_music", "netease", "lastfm"].includes(record.provenance.source_system) &&
           profileImport.source_key === `${record.provenance.source_system}.music_import`)) ||
       !profileImport.member_names.includes(record.provenance.source_member) ||
       record.provenance.profile_import_id?.toLowerCase() !== profileImportId ||
@@ -535,9 +600,45 @@ function ingestionStatements(database) {
         cursor_after_ms = MAX(source_cursors.cursor_after_ms, excluded.cursor_after_ms),
         updated_at = excluded.updated_at`,
     ),
+    hasEvent: database.prepare(
+      "SELECT 1 FROM listening_events WHERE deduplication_key = ?",
+    ),
+    touchCursor: database.prepare(
+      "UPDATE source_cursors SET updated_at = MAX(updated_at, ?) WHERE source_key = ?",
+    ),
   };
 }
 
+// Records the span between the newest stored play and `before` as possibly missed.
+function recordCoverageGap(database, { subjectId, sourceKey, before, detectedAt }) {
+  const after = database.prepare(
+    "SELECT MAX(occurred_at) AS occurred_at FROM listening_events WHERE subject_id = ? AND occurred_at < ?",
+  ).get(subjectId, before)?.occurred_at;
+  // A history that starts with this read has nothing to have missed.
+  if (!after) return;
+  database.prepare(
+    `INSERT OR IGNORE INTO listening_coverage_gaps (subject_id, source_key, gap_start, gap_end, detected_at)
+    VALUES (?, ?, ?, ?, ?)`,
+  ).run(subjectId, sourceKey, after, before, detectedAt);
+}
+
+// Recent syncs from before gaps were recorded: a run that stored a full window of 50
+// new plays at once may have missed plays before its oldest one.
+function recordEarlierRecentSyncGaps(database) {
+  const runs = database.prepare(
+    `SELECT subject_id, json_extract(record_json, '$.ingested_at') AS ingested_at,
+      MIN(occurred_at) AS earliest, COUNT(*) AS plays
+    FROM listening_events
+    WHERE json_extract(record_json, '$.provenance.external_ref.entity_type') = 'spotify.play_history'
+    GROUP BY subject_id, ingested_at HAVING plays >= 50`,
+  ).all();
+  for (const run of runs) {
+    recordCoverageGap(database, {
+      subjectId: run.subject_id, sourceKey: "spotify.recently_played", before: run.earliest, detectedAt: run.ingested_at,
+    });
+  }
+}
+
 function insertBundleRecords(bundle, statements) {
   let insertedTracks = 0;
   let insertedEvents = 0;
@@ -569,9 +670,180 @@ function insertBundleRecords(bundle, statements) {
   return { insertedTracks, insertedEvents };
 }
 
+// Where stored plays came from, in words the agent can repeat. Others keep their kind.
+const COVERAGE_SOURCE_LABELS = {
+  "spotify.play_history": "Spotify recent plays, read while Moondog runs",
+  "spotify.extended_history.play": "Spotify Extended Streaming History file",
+  "spotify.account_data.play_history": "Spotify Account Data file",
+  "listenbrainz.listen": "ListenBrainz export file",
+};
+
+const SCROBBLE_SOURCE_KEY = "lastfm.music_import";
+// Scrobbles record when a song started. Spotify records when a play ended, so its start is
+// the end minus the listening time; Account Data times are rounded to the minute.
+const MIRROR_TOLERANCE_MS = 150_000;
+const DAY_MS = 86_400_000;
+
+// A local calendar date, such as 2026-10-25.
+function localDate(date) {
+  const pad = (value) => String(value).padStart(2, "0");
+  return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`;
+}
+
+const foldLabel = (value) => String(value ?? "").normalize("NFKC").toLocaleLowerCase("und")
+  .replace(/[^\p{L}\p{N}]+/gu, " ").trim();
+const lexicalCompare = (left, right) => (left < right ? -1 : left > right ? 1 : 0);
+
+/**
+ * Many Last.fm scrobbles repeat plays that Spotify or ListenBrainz recorded too. Each scrobble
+ * that matches another service's play (same artist and title, starting within a few minutes)
+ * is linked to it once, whichever was imported first, so the play counts once.
+ */
+function reconcileScrobbleMirrors(database, subjectId, earliest, latest, createdAt) {
+  if (!subjectId || !earliest || !latest) return 0;
+  const scrobbled = database.prepare(
+    "SELECT 1 FROM import_batches WHERE subject_id = ? AND source_key = ? LIMIT 1",
+  ).get(subjectId, SCROBBLE_SOURCE_KEY);
+  if (!scrobbled) return 0;
+  const rows = database.prepare(
+    `SELECT e.listening_event_id AS id, e.occurred_at AS at,
+      e.record_json AS event_json, t.record_json AS track_json
+    FROM listening_events e
+    JOIN track_refs t
+      ON t.track_ref_id = e.track_ref_id AND t.revision = e.track_ref_revision
+    WHERE e.subject_id = ? AND e.occurred_at BETWEEN ? AND ?
+      AND NOT EXISTS (
+        SELECT 1 FROM listening_event_supersessions s
+        WHERE s.superseded_event_id = e.listening_event_id
+      )
+      AND NOT EXISTS (
+        SELECT 1 FROM listening_event_mirrors m
+        WHERE m.mirror_event_id = e.listening_event_id
+          OR m.original_event_id = e.listening_event_id
+      )`,
+  ).all(
+    subjectId,
+    new Date(Date.parse(earliest) - DAY_MS).toISOString(),
+    new Date(Date.parse(latest) + DAY_MS).toISOString(),
+  );
+  const scrobbles = new Map();
+  const plays = [];
+  for (const row of rows) {
+    const event = JSON.parse(row.event_json);
+    const track = JSON.parse(row.track_json);
+    const artist = foldLabel(track.artist_credits?.[0]?.name);
+    const title = foldLabel(track.title);
+    if (!artist || !title) continue;
+    const key = `${artist}\u0000${title}`;
+    const at = Date.parse(row.at);
+    const system = String(event.provenance?.external_ref?.system ?? "");
+    if (system === "lastfm") {
+      if (!scrobbles.has(key)) scrobbles.set(key, []);
+      scrobbles.get(key).push({ id: row.id, start: at });
+      continue;
+    }
+    const length = Number.isSafeInteger(event.played_ms) ? event.played_ms
+      : Number.isSafeInteger(track.duration_ms) ? track.duration_ms : null;
+    // A Spotify play of unknown length can't be placed precisely enough to match.
+    if (system.startsWith("spotify") && length === null) continue;
+    plays.push({ id: row.id, key, start: system.startsWith("spotify") ? at - length : at });
+  }
+  const pairs = [];
+  for (const list of scrobbles.values()) list.sort((left, right) => left.start - right.start);
+  for (const play of plays) {
+    const list = scrobbles.get(play.key);
+    if (!list) continue;
+    let low = 0;
+    let high = list.length;
+    while (low < high) {
+      const middle = (low + high) >> 1;
+      if (list[middle].start < play.start - MIRROR_TOLERANCE_MS) low = middle + 1;
+      else high = middle;
+    }
+    for (let index = low; index < list.length && list[index].start <= play.start + MIRROR_TOLERANCE_MS; index += 1) {
+      pairs.push({ gap: Math.abs(list[index].start - play.start), play: play.id, scrobble: list[index].id });
+    }
+  }
+  // Closest pairs first, so each play and each scrobble is used once.
+  pairs.sort((left, right) => left.gap - right.gap ||
+    lexicalCompare(left.scrobble, right.scrobble) || lexicalCompare(left.play, right.play));
+  const insert = database.prepare(
+    `INSERT INTO listening_event_mirrors (mirror_event_id, original_event_id, created_at)
+    VALUES (?, ?, ?)`,
+  );
+  const usedPlays = new Set();
+  const usedScrobbles = new Set();
+  let linked = 0;
+  for (const pair of pairs) {
+    if (usedPlays.has(pair.play) || usedScrobbles.has(pair.scrobble)) continue;
+    usedPlays.add(pair.play);
+    usedScrobbles.add(pair.scrobble);
+    linked += insert.run(pair.scrobble, pair.play, createdAt).changes;
+  }
+  return linked;
+}
+
+function preserveExtendedHistoryMatches(bundle, retainedMatch) {
+  const candidates = bundle.reconciliationCandidates;
+  if (bundle.importBatch.sourceFormat !== "spotify_extended_streaming_history_music_v1") return candidates;
+  const retained = new Map(candidates.map((candidate) => [
+    candidate.supersedingEventId, retainedMatch(candidate.supersedingEventId),
+  ]));
+  if (!candidates.some((candidate) => retained.get(candidate.supersedingEventId) &&
+    retained.get(candidate.supersedingEventId) !== candidate.supersededEventId)) return candidates;
+
+  // Account Data cannot distinguish equal-duration plays of the same title and
+  // artist within one minute. Added video plays can reorder those ordinals.
+  // A narrower export may omit an established counterpart's ordinal entirely.
+  // Keep those pairs too; only new plays use this archive's remaining candidates.
+  // The store's conflict checks still reject a new play claiming an absent play's
+  // retained counterpart, so an incomplete export cannot invent another match.
+  const events = new Map(bundle.listeningEvents.map((event) => [event.listening_event_id, event]));
+  const tracks = new Map(bundle.trackRefs.map((track) => [track.track_ref_id, track]));
+  const groups = new Map();
+  for (const candidate of candidates) {
+    const event = events.get(candidate.supersedingEventId);
+    const track = tracks.get(event.track_ref_id);
+    const key = JSON.stringify([
+      event.occurred_at.slice(0, 16), event.played_ms,
+      track.title.normalize("NFKC").toLowerCase(),
+      track.artist_credits[0].name.normalize("NFKC").toLowerCase(),
+    ]);
+    const group = groups.get(key) ?? [];
+    group.push(candidate);
+    groups.set(key, group);
+  }
+  const matches = new Map();
+  for (const group of groups.values()) {
+    const available = new Set(group.map((candidate) => candidate.supersededEventId));
+    for (const candidate of group) {
+      const previous = retained.get(candidate.supersedingEventId);
+      if (!previous) continue;
+      available.delete(previous);
+      matches.set(candidate.supersedingEventId, previous);
+    }
+    const remaining = available.values();
+    for (const candidate of group) {
+      if (matches.has(candidate.supersedingEventId)) continue;
+      const selected = available.has(candidate.supersededEventId)
+        ? candidate.supersededEventId : remaining.next().value;
+      matches.set(candidate.supersedingEventId, selected);
+      available.delete(selected);
+    }
+  }
+  return candidates.map((candidate) => ({
+    ...candidate, supersededEventId: matches.get(candidate.supersedingEventId),
+  }));
+}
+
 export class ListeningHistoryStore {
   #database;
   #closed = false;
+  #snapshot = null;
+  #projection = null;
+  #coverageSources = null;
+  #evidence = null;
+  #ledgerChanges = 0;
 
   constructor(database) {
     this.#database = database;
@@ -670,6 +942,10 @@ export class ListeningHistoryStore {
           AND NOT EXISTS (
             SELECT 1 FROM listening_event_supersessions s
             WHERE s.superseded_event_id = e.listening_event_id
+          )
+          AND NOT EXISTS (
+            SELECT 1 FROM listening_event_mirrors m
+            WHERE m.mirror_event_id = e.listening_event_id
           )`,
       )
       .get(normalized).count;
@@ -1009,6 +1285,22 @@ export class ListeningHistoryStore {
       });
   }
 
+  // How many choices are in force, beyond the ones a list shows.
+  countActiveListenerCorrections({ subjectId } = {}) {
+    if (this.#closed) throw new Error("Listening history store is closed");
+    if (!isUuid(subjectId)) {
+      throw new TypeError("A valid listener correction subject is required");
+    }
+    return this.#database
+      .prepare(
+        `SELECT COUNT(*) AS count FROM taste_events a
+        WHERE a.subject_id = ? AND a.operation = 'assert'
+          AND NOT EXISTS (SELECT 1 FROM taste_events r WHERE r.retracts_taste_event_id = a.taste_event_id)
+          AND NOT EXISTS (SELECT 1 FROM taste_events s WHERE s.supersedes_taste_event_id = a.taste_event_id)`,
+      )
+      .get(subjectId.toLowerCase()).count;
+  }
+
   ingest(value) {
     if (this.#closed) throw new Error("Listening history store is closed");
     const bundle = normalizedBundle(value);
@@ -1024,14 +1316,24 @@ export class ListeningHistoryStore {
     this.#database.exec("BEGIN IMMEDIATE");
     let insertedTracks;
     let insertedEvents;
+    let mirroredEvents = 0;
     try {
       if (subjectId !== undefined) {
         this.localSubjectId({ preferredSubjectId: subjectId });
       }
+      const times = bundle.listeningEvents.map((record) => record.occurred_at).sort();
+      // A read that overlaps a stored play joins up with it.
+      const overlaps = bundle.listeningEvents.some((record) => statements.hasEvent.get(record.deduplication_key));
+      if (bundle.windowFull && subjectId !== undefined && !overlaps) {
+        recordCoverageGap(this.#database, { subjectId, sourceKey: bundle.sourceKey, before: times[0], detectedAt: bundle.capturedAt });
+      }
       ({ insertedTracks, insertedEvents } = insertBundleRecords(
         bundle,
         statements,
       ));
+      // A read that finds nothing new still moves the source's checked time.
+      statements.touchCursor.run(bundle.capturedAt, bundle.sourceKey);
+      mirroredEvents = reconcileScrobbleMirrors(this.#database, subjectId, times[0], times.at(-1), bundle.capturedAt);
       this.#database.exec("COMMIT");
     } catch (error) {
       this.#database.exec("ROLLBACK");
@@ -1045,6 +1347,7 @@ export class ListeningHistoryStore {
       inserted_events: insertedEvents,
       duplicate_events: bundle.listeningEvents.length - insertedEvents,
       inserted_track_refs: insertedTracks,
+      ...(mirroredEvents ? { mirrored_events: mirroredEvents } : {}),
       cursor_after_ms: this.sourceCursor(bundle.sourceKey),
       profile_effects: insertedEvents > 0 ? "updated" : "unchanged",
     };
@@ -1055,7 +1358,7 @@ export class ListeningHistoryStore {
     const bundle = normalizedImportBundle(value);
     const batch = bundle.importBatch;
     const existingBatch = this.#database.prepare(
-      `SELECT import_batch_id FROM import_batches
+      `SELECT * FROM import_batches
       WHERE import_batch_id = ?`,
     );
     const insertBatch = this.#database.prepare(
@@ -1141,12 +1444,14 @@ export class ListeningHistoryStore {
     let insertedTracks = 0;
     let insertedEvents = 0;
     let supersededEvents = 0;
+    let mirroredEvents = 0;
     let insertedProfileEvidence = 0;
     let alreadyImported = false;
     let profileAlreadyImported = bundle.profileImport === undefined;
     try {
       this.localSubjectId({ preferredSubjectId: batch.subjectId });
-      alreadyImported = Boolean(existingBatch.get(batch.importBatchId));
+      const previousBatch = existingBatch.get(batch.importBatchId);
+      alreadyImported = Boolean(previousBatch);
       if (!alreadyImported) {
         ({ insertedTracks, insertedEvents } = insertBundleRecords(
           bundle,
@@ -1169,8 +1474,38 @@ export class ListeningHistoryStore {
           batch.inputRecords - insertedEvents,
           insertedTracks,
         );
+      } else {
+        // Earlier readers skipped video members, even when they contained music.
+        // Extend that same archive's receipt without replacing its existing rows.
+        const previousMembers = JSON.parse(previousBatch.member_names_json);
+        const addedMembers = batch.memberNames.filter((name) => !previousMembers.includes(name));
+        const expandsExtendedArchive = batch.sourceFormat === "spotify_extended_streaming_history_music_v1" &&
+          previousBatch.subject_id === batch.subjectId && previousBatch.source_key === bundle.sourceKey &&
+          previousBatch.source_format === batch.sourceFormat && previousBatch.data_scope === batch.dataScope &&
+          previousBatch.archive_sha256 === batch.archiveSha256 && previousBatch.archive_size_bytes === batch.archiveSizeBytes &&
+          previousMembers.every((name) => batch.memberNames.includes(name)) && addedMembers.length > 0 &&
+          addedMembers.every((name) => /^Streaming_History_Video_\d{4}(?:_\d+)?\.json$/u.test(name));
+        if (expandsExtendedArchive) {
+          const incomingIds = new Set(bundle.listeningEvents.map((event) => event.listening_event_id));
+          const previousEvents = this.#database.prepare(`SELECT listening_event_id FROM listening_events
+            WHERE subject_id = ? AND json_extract(record_json, '$.provenance.import_batch_id') = ?`)
+            .all(batch.subjectId, batch.importBatchId);
+          if (batch.inputRecords < previousBatch.input_records || previousEvents.some((event) => !incomingIds.has(event.listening_event_id))) {
+            throw new Error("Extended archive expansion must preserve every previously imported event");
+          }
+          ({ insertedTracks, insertedEvents } = insertBundleRecords(bundle, statements));
+          this.#database.prepare(`UPDATE import_batches SET member_names_json = ?, input_records = ?,
+            earliest_occurred_at = ?, latest_occurred_at = ?,
+            inserted_events = inserted_events + ?, duplicate_events = ? - inserted_events - ?,
+            inserted_track_refs = inserted_track_refs + ? WHERE import_batch_id = ?`)
+            .run(JSON.stringify(batch.memberNames), batch.inputRecords, batch.earliestOccurredAt, batch.latestOccurredAt,
+              insertedEvents, batch.inputRecords, insertedEvents, insertedTracks, batch.importBatchId);
+        }
       }
-      for (const candidate of bundle.reconciliationCandidates) {
+      const reconciliationCandidates = preserveExtendedHistoryMatches(bundle, (eventId) =>
+        selectCandidateBySuperseding.get(eventId)?.superseded_event_id ??
+        selectBySuperseding.get(eventId)?.superseded_event_id);
+      for (const candidate of reconciliationCandidates) {
         const existingBySuperseded = selectCandidateBySuperseded.get(
           candidate.supersededEventId,
         )?.superseding_event_id;
@@ -1292,6 +1627,9 @@ export class ListeningHistoryStore {
           );
         }
       }
+      mirroredEvents = reconcileScrobbleMirrors(
+        this.#database, batch.subjectId, batch.earliestOccurredAt, batch.latestOccurredAt, batch.importedAt,
+      );
       this.#database.exec("COMMIT");
     } catch (error) {
       this.#database.exec("ROLLBACK");
@@ -1310,7 +1648,8 @@ export class ListeningHistoryStore {
       duplicate_events: batch.inputRecords - insertedEvents,
       inserted_track_refs: insertedTracks,
       superseded_events: supersededEvents,
-      effective_event_delta: insertedEvents - supersededEvents,
+      ...(mirroredEvents ? { mirrored_events: mirroredEvents } : {}),
+      effective_event_delta: insertedEvents - supersededEvents - mirroredEvents,
       cursor_after_ms: this.sourceCursor(bundle.sourceKey),
       earliest_occurred_at: batch.earliestOccurredAt,
       latest_occurred_at: batch.latestOccurredAt,
@@ -1333,7 +1672,7 @@ export class ListeningHistoryStore {
           }
         : {}),
       profile_effects:
-        insertedEvents - supersededEvents !== 0 || insertedProfileEvidence > 0
+        insertedEvents > 0 || supersededEvents > 0 || insertedProfileEvidence > 0
           ? "updated"
           : "unchanged",
     };
@@ -1355,6 +1694,10 @@ export class ListeningHistoryStore {
             SELECT 1 FROM listening_event_supersessions
             WHERE superseded_event_id = listening_events.listening_event_id
           )
+          AND NOT EXISTS (
+            SELECT 1 FROM listening_event_mirrors m
+            WHERE m.mirror_event_id = listening_events.listening_event_id
+          )
         ORDER BY occurred_at DESC, listening_event_id DESC
         LIMIT ?`,
       )
@@ -1454,6 +1797,10 @@ export class ListeningHistoryStore {
             SELECT 1 FROM listening_event_supersessions s
             WHERE s.superseded_event_id = e.listening_event_id
           )
+          AND NOT EXISTS (
+            SELECT 1 FROM listening_event_mirrors m
+            WHERE m.mirror_event_id = e.listening_event_id
+          )
         ORDER BY e.occurred_at, e.listening_event_id`,
       )
       .all(normalizedSubjectId)
@@ -1487,15 +1834,103 @@ export class ListeningHistoryStore {
       )
       .all(normalizedSubjectId)
       .map((row) => JSON.parse(row.record_json));
-    return projectListeningProfile({
-      subjectId: normalizedSubjectId,
-      eventRows,
-      profileEvidence,
-      tasteEvents,
-      maxItems,
+    const { rows, adopted } = markMoondogStartedPlays(eventRows, this.#suggestionsForMatching());
+    return {
+      ...projectListeningProfile({
+        subjectId: normalizedSubjectId,
+        eventRows: rows,
+        profileEvidence,
+        tasteEvents,
+        maxItems,
+        adoptedSuggestionIds: adopted,
+      }),
+      adoptedSuggestionIds: adopted,
+    };
+  }
+
+  #suggestionsForMatching() {
+    return this.#database.prepare(`SELECT suggestion_id, provider_track_id, isrc, track_key, release_key, title, artist_credit,
+      requested_at FROM moondog_suggestions ORDER BY requested_at, suggestion_id`).all();
+  }
+
+  /** Records what Moondog itself queued or played. Only accepted host writes belong here. */
+  recordMoondogSuggestions({ entries } = {}) {
+    if (this.#closed) throw new Error("Listening history store is closed");
+    if (!Array.isArray(entries) || entries.length > 50) throw new TypeError("Moondog suggestions are invalid");
+    const insert = this.#database.prepare(`INSERT INTO moondog_suggestions
+      (suggestion_id, service, action, provider_track_id, isrc, track_key, release_key, title, artist_credit, requested_at, mix_slot)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
+    let recorded = 0;
+    this.#database.exec("BEGIN IMMEDIATE");
+    try {
+      for (const value of entries) {
+        const entry = suggestionEntry(value);
+        if (!entry || !/^[a-z][a-z0-9_.]{1,63}$/u.test(entry.action ?? "") || !Number.isFinite(Date.parse(entry.requested_at))) continue;
+        insert.run(randomUUID(), entry.service, entry.action, entry.provider_track_id, entry.isrc, entry.track_key, entry.release_key,
+          entry.title, entry.artist_credit, new Date(entry.requested_at).toISOString(), entry.mix_slot);
+        recorded += 1;
+      }
+      this.#database.exec("COMMIT");
+    } catch (error) {
+      this.#database.exec("ROLLBACK");
+      throw error;
+    }
+    this.#ledgerChanges += recorded;
+    return { recorded };
+  }
+
+  /**
+   * Moondog's earlier suggestions of each track, matched as stored plays are: the same provider
+   * track, ISRC or song. Adopted means the listener later played it again on their own.
+   */
+  moondogSuggestionHistory({ subjectId, tracks } = {}) {
+    if (!Array.isArray(tracks)) throw new TypeError("Suggestion history tracks are invalid");
+    const suggestions = this.#suggestionsForMatching();
+    if (!suggestions.length) return tracks.map(() => null);
+    const { adopted } = this.#listeningSnapshot(subjectId);
+    const byIdentity = new Map();
+    for (const suggestion of suggestions) {
+      for (const identity of suggestionIdentities(suggestion)) {
+        byIdentity.set(identity, [...(byIdentity.get(identity) ?? []), suggestion]);
+      }
+    }
+    return tracks.map((track) => {
+      const entry = suggestionEntry({ service: "spotify", action: "lookup", ...track });
+      const matches = new Set(entry ? suggestionIdentities(entry).flatMap((identity) => byIdentity.get(identity) ?? []) : []);
+      if (!matches.size) return null;
+      const last = [...matches].reduce((latest, item) => item.requested_at > latest ? item.requested_at : latest, "");
+      return { times: matches.size, last_suggested_at: last.slice(0, 10),
+        adopted: [...matches].some((item) => adopted.has(item.suggestion_id)) };
     });
   }
 
+  /** Where each track sits in the listener's own plays (MIX_SLOTS in moondog-suggestions.mjs). */
+  listeningMix({ subjectId, tracks } = {}) {
+    if (!Array.isArray(tracks)) throw new TypeError("Listening mix tracks are invalid");
+    const { familiarity } = this.#listeningSnapshot(subjectId);
+    return tracks.map((track) => mixSlot(track, familiarity));
+  }
+
+  // One projection with a single item per list serves the status, the Avoids, the catalog, the
+  // mix and the suggestion history until the listening data changes. Callers copy what they return.
+  #sharedProjection(subjectId) {
+    const version = `${subjectId}:${this.listeningDataVersion()}`;
+    if (this.#projection?.version !== version) {
+      this.#projection = { version, value: this.#profileProjection({ subjectId, maxItems: 1 }) };
+    }
+    return this.#projection.value;
+  }
+
+  #listeningSnapshot(subjectId) {
+    const version = `${subjectId}:${this.listeningDataVersion()}`;
+    if (this.#snapshot?.version !== version) {
+      const projected = this.#sharedProjection(subjectId);
+      this.#snapshot = { version, adopted: projected.adoptedSuggestionIds,
+        familiarity: listeningFamiliarity(projected.catalog, projected.summary.listening_behavior.context.reference_date) };
+    }
+    return this.#snapshot;
+  }
+
   /** Every track the subject has played, with its play count, most played first. */
   listenedTracks({ subjectId } = {}) {
     if (this.#closed) throw new Error("Listening history store is closed");
@@ -1517,6 +1952,10 @@ export class ListeningHistoryStore {
             SELECT 1 FROM listening_event_supersessions
             WHERE superseded_event_id = listening_events.listening_event_id
           )
+          AND NOT EXISTS (
+            SELECT 1 FROM listening_event_mirrors m
+            WHERE m.mirror_event_id = listening_events.listening_event_id
+          )
         GROUP BY track_ref_id
         ORDER BY play_count DESC, track_ref_id`,
       )
@@ -1545,10 +1984,186 @@ export class ListeningHistoryStore {
     return structuredClone({ ...value.summary, lyric_exclusions: value.lyricExclusions });
   }
 
+  activeAvoidances({ subjectId } = {}) {
+    // Host-side filtering must include corrections outside the bounded display.
+    return structuredClone(this.#sharedProjection(subjectId).lyricExclusions);
+  }
+
   profileSummary({ subjectId, maxItems = 10 } = {}) {
-    return structuredClone(
-      this.#profileProjection({ subjectId, maxItems }).summary,
-    );
+    return structuredClone((maxItems === 1 ? this.#sharedProjection(subjectId)
+      : this.#profileProjection({ subjectId, maxItems })).summary);
+  }
+
+  profileCatalog({ subjectId } = {}) {
+    const projected = this.#sharedProjection(subjectId);
+    const normalizedSubjectId = subjectId.toLowerCase();
+    const imports = this.#database.prepare(`SELECT source_format AS format, data_scope AS scope,
+      SUM(input_records) AS input_records, MIN(earliest_occurred_at) AS earliest,
+      MAX(latest_occurred_at) AS latest
+      FROM import_batches WHERE subject_id = ? GROUP BY source_format, data_scope ORDER BY source_format`)
+      .all(normalizedSubjectId);
+    const stored = this.#database.prepare("SELECT COUNT(*) AS count FROM listening_events WHERE subject_id = ?")
+      .get(normalizedSubjectId).count;
+    return structuredClone({
+      coverage: { ...projected.summary.coverage,
+        stored_listening_events: stored,
+        superseded_listening_events: stored - projected.summary.coverage.effective_listening_events },
+      sources: imports,
+      context: {
+        reference_date: projected.summary.listening_behavior.context.reference_date,
+        recent_window_days: projected.summary.listening_behavior.context.recent_window_days,
+        incognito_events_excluded: projected.summary.listening_behavior.context.incognito_events_excluded,
+        ...(projected.summary.listening_behavior.context.moondog_started_events_excluded
+          ? { moondog_started_events_excluded: projected.summary.listening_behavior.context.moondog_started_events_excluded } : {}),
+        release_minimum_distinct_tracks: projected.summary.listening_behavior.context.release_minimum_distinct_tracks,
+      },
+      sections: projected.catalog,
+    });
+  }
+
+  // What the listener played lately: stored plays on today and the `days` - 1 local days before
+  // it, by artist. It reads the clock, not the newest play, so a history that stopped shows as
+  // quiet. Whole local days keep the window, and so the model's context, the same all day.
+  recentAttention({ subjectId, days = 28, now = Date.now(), limit = 10 } = {}) {
+    if (this.#closed) throw new Error("Listening history store is closed");
+    if (!isUuid(subjectId)) throw new TypeError("A valid profile subject is required");
+    const today = new Date(now);
+    const start = new Date(today);
+    start.setHours(0, 0, 0, 0);
+    start.setDate(start.getDate() - (days - 1));
+    const from = start.toISOString();
+    const effective = `e.subject_id = ? AND e.occurred_at >= ?
+      AND NOT EXISTS (SELECT 1 FROM listening_event_supersessions s WHERE s.superseded_event_id = e.listening_event_id)
+      AND NOT EXISTS (SELECT 1 FROM listening_event_mirrors m WHERE m.mirror_event_id = e.listening_event_id)`;
+    const subject = subjectId.toLowerCase();
+    // Plays Moondog started are not the listener's own attention.
+    const suggestions = this.#suggestionsForMatching()
+      .filter((suggestion) => Date.parse(suggestion.requested_at) + SUGGESTION_WINDOW.afterMs >= Date.parse(from));
+    const started = suggestions.length === 0 ? [] : markMoondogStartedPlays(this.#database.prepare(`SELECT
+      e.listening_event_id, e.record_json AS event_json, t.record_json AS track_json FROM listening_events e
+      JOIN track_refs t ON t.track_ref_id = e.track_ref_id AND t.revision = e.track_ref_revision WHERE ${effective}`)
+      .all(subject, from).map((row) => ({ id: row.listening_event_id, event: JSON.parse(row.event_json), track: JSON.parse(row.track_json) })),
+    suggestions).rows.filter((row) => row.moondog_suggestion_id).map((row) => row.id);
+    const own = `${effective} AND e.listening_event_id NOT IN (SELECT value FROM json_each(?))`;
+    const plays = this.#database.prepare(`SELECT COUNT(*) AS plays FROM listening_events e WHERE ${own}`)
+      .get(subject, from, JSON.stringify(started)).plays;
+    const artists = this.#database.prepare(`SELECT json_extract(a.value, '$.name') AS name, COUNT(*) AS plays,
+      MAX(e.occurred_at) AS last_played_at
+      FROM listening_events e
+      JOIN track_refs t ON t.track_ref_id = e.track_ref_id AND t.revision = e.track_ref_revision
+      JOIN json_each(t.record_json, '$.artist_credits') a
+      WHERE ${own} AND name IS NOT NULL
+      GROUP BY name ORDER BY plays DESC, last_played_at DESC LIMIT ?`).all(subject, from, JSON.stringify(started), limit);
+    return structuredClone({ from_date: localDate(start), through_date: localDate(today), plays,
+      ...(started.length ? { moondog_started_plays: started.length } : {}), top_artists: artists });
+  }
+
+  // Which spans of time the stored plays cover, and where plays may be missing.
+  listeningCoverage({ subjectId } = {}) {
+    if (this.#closed) throw new Error("Listening history store is closed");
+    if (!isUuid(subjectId)) throw new TypeError("A valid profile subject is required");
+    const subject = subjectId.toLowerCase();
+    // Reading every play is the slow part, so it is kept until the listening data changes.
+    const version = `${subject}:${this.listeningDataVersion()}`;
+    if (this.#coverageSources?.version !== version) {
+      this.#coverageSources = { version, sources: this.#database.prepare(`SELECT
+        json_extract(record_json, '$.provenance.external_ref.entity_type') AS kind,
+        COUNT(*) AS plays, MIN(occurred_at) AS first_played_at, MAX(occurred_at) AS last_played_at
+        FROM listening_events WHERE subject_id = ? GROUP BY kind ORDER BY first_played_at`).all(subject) };
+    }
+    const { sources } = this.#coverageSources;
+    // A history file imported later can fill a span that a sync missed.
+    const filled = this.#database.prepare(`SELECT earliest_occurred_at AS earliest, latest_occurred_at AS latest
+      FROM import_batches WHERE subject_id = ? AND earliest_occurred_at IS NOT NULL AND latest_occurred_at IS NOT NULL`).all(subject);
+    const gaps = this.#database.prepare(`SELECT gap_start AS "from", gap_end AS "to"
+      FROM listening_coverage_gaps WHERE subject_id = ? ORDER BY gap_start`).all(subject)
+      .flatMap((gap) => filled.reduce((spans, { earliest, latest }) => spans.flatMap((span) =>
+        latest <= span.from || earliest >= span.to ? [span] : [
+          ...(earliest > span.from ? [{ from: span.from, to: earliest }] : []),
+          ...(latest < span.to ? [{ from: latest, to: span.to }] : []),
+        ]), [{ ...gap }]));
+    const checked = this.#database.prepare("SELECT updated_at FROM source_cursors WHERE source_key = 'spotify.recently_played'").get();
+    return structuredClone({
+      sources: sources.map(({ kind, ...source }) => ({ source: COVERAGE_SOURCE_LABELS[kind] ?? kind, ...source })),
+      gaps: gaps.slice(-10),
+      ...(checked ? { recent_plays_checked_at: checked.updated_at } : {}),
+    });
+  }
+
+  listenerProfileStorage({ subjectId } = {}) {
+    if (this.#closed) throw new Error("Listening history store is closed");
+    if (!isUuid(subjectId)) throw new TypeError("A valid profile subject is required");
+    return new ListenerProfileStore(this.#database, subjectId.toLowerCase());
+  }
+
+  profileDataVersion() {
+    if (this.#closed) throw new Error("Listening history store is closed");
+    // data_version observes other connections; total_changes observes our own.
+    // This is a connection-local cache key, never a persisted evidence digest.
+    return `${this.#database.prepare("PRAGMA data_version").get().data_version}:${
+      this.#database.prepare("SELECT total_changes() AS changes").get().changes}`;
+  }
+
+  // A version of the listening evidence that no suggestions ledger write changes, from this
+  // connection or another, such as the other surface's. A new suggestion changes no projection
+  // until its plays arrive, so caches of the projection, the Avoids, the status and the build
+  // input key on this and are not rebuilt after every queue. This connection's own writes count
+  // by total_changes. Another connection's commit moves data_version whatever it wrote, so then
+  // the evidence tables themselves say whether the evidence changed.
+  listeningDataVersion() {
+    if (this.#closed) throw new Error("Listening history store is closed");
+    const dataVersion = this.#database.prepare("PRAGMA data_version").get().data_version;
+    const ownChanges = this.#database.prepare("SELECT total_changes() AS changes").get().changes - this.#ledgerChanges;
+    if (this.#evidence?.dataVersion !== dataVersion || this.#evidence.ownChanges !== ownChanges) {
+      this.#evidence = { dataVersion, ownChanges, fingerprint: this.#evidenceFingerprint() };
+    }
+    return this.#evidence.fingerprint;
+  }
+
+  #evidenceFingerprint() {
+    const digest = createHash("sha256");
+    for (const table of APPENDED_EVIDENCE_TABLES) {
+      digest.update(JSON.stringify(this.#database.prepare(`SELECT COUNT(*) AS rows, MAX(rowid) AS last FROM ${table}`).get()));
+    }
+    for (const table of UPDATED_EVIDENCE_TABLES) {
+      for (const row of this.#database.prepare(`SELECT * FROM ${table} ORDER BY rowid`).iterate()) digest.update(JSON.stringify(row));
+    }
+    return digest.digest("hex");
+  }
+
+  profileEvidenceRevision({ subjectId } = {}) {
+    if (this.#closed) throw new Error("Listening history store is closed");
+    if (!isUuid(subjectId)) throw new TypeError("A valid profile subject is required");
+    const digest = createHash("sha256");
+    // Build artifacts never feed back into source evidence. Hash locally; raw
+    // events, source identifiers and private import metadata do not go to Pi.
+    const queries = [
+      "SELECT record_json FROM listening_events WHERE subject_id = ? ORDER BY listening_event_id",
+      "SELECT record_json FROM spotify_profile_evidence WHERE subject_id = ? ORDER BY profile_evidence_id",
+      "SELECT record_json FROM taste_events WHERE subject_id = ? ORDER BY taste_event_id",
+      "SELECT * FROM import_batches WHERE subject_id = ? ORDER BY import_batch_id",
+      "SELECT * FROM profile_import_batches WHERE subject_id = ? ORDER BY profile_import_id",
+      `SELECT t.record_json FROM track_refs t WHERE EXISTS (SELECT 1 FROM listening_events e
+        WHERE e.subject_id = ? AND e.track_ref_id = t.track_ref_id AND e.track_ref_revision = t.revision)
+        ORDER BY t.track_ref_id, t.revision`,
+    ];
+    for (const query of queries) {
+      digest.update(query).update("\0");
+      for (const row of this.#database.prepare(query).iterate(subjectId.toLowerCase())) {
+        digest.update(JSON.stringify(row)).update("\0");
+      }
+    }
+    // Moondog's suggestions decide which plays count as the listener's own. A play can match a
+    // suggestion made up to SUGGESTION_WINDOW.beforeMs after it, for clock skew; one made later
+    // than that after the latest stored play matches nothing yet, so it leaves the reading current.
+    const latest = this.#database.prepare("SELECT MAX(occurred_at) AS latest FROM listening_events WHERE subject_id = ?")
+      .get(subjectId.toLowerCase()).latest;
+    const latestMs = Date.parse(latest ?? "");
+    const suggestions = Number.isFinite(latestMs) ? this.#database.prepare(`SELECT suggestion_id, provider_track_id, isrc, track_key,
+      release_key, title, artist_credit, requested_at FROM moondog_suggestions WHERE requested_at <= ? ORDER BY suggestion_id`)
+      .all(new Date(latestMs + SUGGESTION_WINDOW.beforeMs).toISOString()) : [];
+    if (suggestions.length) digest.update("moondog_suggestions\0").update(JSON.stringify(suggestions));
+    return digest.digest("hex");
   }
 
   rediscoveryCandidates({ subjectId, limit = 6 } = {}) {
@@ -1930,6 +2545,10 @@ export class ListeningHistoryStore {
           WHERE NOT EXISTS (
             SELECT 1 FROM listening_event_supersessions
             WHERE superseded_event_id = listening_events.listening_event_id
+          )
+          AND NOT EXISTS (
+            SELECT 1 FROM listening_event_mirrors m
+            WHERE m.mirror_event_id = listening_events.listening_event_id
           )`,
         )
         .get().count,
@@ -2003,7 +2622,11 @@ async function initializeListeningHistoryDatabase(
   ) {
     throw new Error("Listening history database schema is incompatible");
   }
+  const recordsGaps = database.prepare(
+    "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'listening_coverage_gaps'",
+  ).get();
   database.exec(schema);
+  if (!recordsGaps) recordEarlierRecentSyncGaps(database);
   if (currentVersion < LISTENING_HISTORY_SCHEMA_VERSION) {
     database.exec(
       `PRAGMA user_version = ${LISTENING_HISTORY_SCHEMA_VERSION}`,