@audiofool/moondog 0.1.0 → 0.2.0-beta.2

package/src/profile/listening-history-store.mjs

@@ -1,4 +1,4 @@
-import { randomUUID } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 import { chmod, mkdir } from "node:fs/promises";
 import path from "node:path";
 
@@ -9,8 +9,9 @@ import {
   createListenerCorrectionRetraction,
 } from "./listener-corrections.mjs";
 import { projectListeningProfile } from "./listening-profile-projection.mjs";
+import { ListenerProfileStore, listenerProfileSchema } from "./listener-profile-store.mjs";
 
-export const LISTENING_HISTORY_SCHEMA_VERSION = 6;
+export const LISTENING_HISTORY_SCHEMA_VERSION = 7;
 
 export function resolveListeningHistoryPath(environment = process.env) {
   return path.join(resolveMoondogStateDirectory(environment), "listening-history.sqlite");
@@ -48,6 +49,8 @@ const schema = `
 
   CREATE INDEX IF NOT EXISTS listening_events_subject_time
     ON listening_events(subject_id, occurred_at DESC);
+  CREATE INDEX IF NOT EXISTS listening_events_subject_track
+    ON listening_events(subject_id, track_ref_id, track_ref_revision);
 
   CREATE TABLE IF NOT EXISTS source_cursors (
     source_key TEXT PRIMARY KEY NOT NULL,
@@ -55,6 +58,18 @@ const schema = `
     updated_at TEXT NOT NULL
   ) STRICT;
 
+  -- Spotify keeps only a listener's last 50 plays. When a sync finds the window full,
+  -- plays between the last stored one and the oldest returned one may be gone for good.
+  CREATE TABLE IF NOT EXISTS listening_coverage_gaps (
+    subject_id TEXT NOT NULL,
+    source_key TEXT NOT NULL,
+    gap_start TEXT NOT NULL,
+    gap_end TEXT NOT NULL,
+    detected_at TEXT NOT NULL,
+    PRIMARY KEY (subject_id, gap_start, gap_end),
+    CHECK (gap_start < gap_end)
+  ) STRICT;
+
   CREATE TABLE IF NOT EXISTS import_batches (
     import_batch_id TEXT PRIMARY KEY NOT NULL,
     subject_id TEXT NOT NULL,
@@ -98,6 +113,18 @@ const schema = `
       REFERENCES import_batches(import_batch_id)
   ) STRICT;
 
+  -- A Last.fm scrobble that repeats a play another service recorded stays stored,
+  -- linked to that play, and is left out of every count.
+  CREATE TABLE IF NOT EXISTS listening_event_mirrors (
+    mirror_event_id TEXT PRIMARY KEY NOT NULL,
+    original_event_id TEXT NOT NULL UNIQUE,
+    created_at TEXT NOT NULL,
+    FOREIGN KEY (mirror_event_id)
+      REFERENCES listening_events(listening_event_id),
+    FOREIGN KEY (original_event_id)
+      REFERENCES listening_events(listening_event_id)
+  ) STRICT;
+
   CREATE TABLE IF NOT EXISTS profile_import_batches (
     profile_import_id TEXT PRIMARY KEY NOT NULL,
     subject_id TEXT NOT NULL,
@@ -158,6 +185,8 @@ const schema = `
   CREATE UNIQUE INDEX IF NOT EXISTS taste_events_one_retraction
     ON taste_events(retracts_taste_event_id)
     WHERE retracts_taste_event_id IS NOT NULL;
+
+  ${listenerProfileSchema}
 `;
 
 function listenerCorrectionError(code, message) {
@@ -207,6 +236,7 @@ function normalizedBundle(value, maximumRecords = 50) {
     capturedAt: new Date(value.captured_at).toISOString(),
     trackRefs: value.track_refs,
     listeningEvents: value.listening_events,
+    windowFull: value.window_full === true,
   };
 }
 
@@ -321,7 +351,8 @@ function normalizedProfileInput(value, batch) {
       !isPlainObject(record.provenance) ||
       record.provenance.source_kind !== "import" ||
       !(record.provenance.source_system === "spotify_account_data" ||
-        (["youtube_music", "qq_music", "netease"].includes(record.provenance.source_system) &&
+        (record.provenance.source_system === "spotify_web_api" && profileImport.source_key === "spotify.web_api_library") ||
+        (["youtube_music", "qq_music", "netease", "lastfm"].includes(record.provenance.source_system) &&
           profileImport.source_key === `${record.provenance.source_system}.music_import`)) ||
       !profileImport.member_names.includes(record.provenance.source_member) ||
       record.provenance.profile_import_id?.toLowerCase() !== profileImportId ||
@@ -535,9 +566,45 @@ function ingestionStatements(database) {
         cursor_after_ms = MAX(source_cursors.cursor_after_ms, excluded.cursor_after_ms),
         updated_at = excluded.updated_at`,
     ),
+    hasEvent: database.prepare(
+      "SELECT 1 FROM listening_events WHERE deduplication_key = ?",
+    ),
+    touchCursor: database.prepare(
+      "UPDATE source_cursors SET updated_at = MAX(updated_at, ?) WHERE source_key = ?",
+    ),
   };
 }
 
+// Records the span between the newest stored play and `before` as possibly missed.
+function recordCoverageGap(database, { subjectId, sourceKey, before, detectedAt }) {
+  const after = database.prepare(
+    "SELECT MAX(occurred_at) AS occurred_at FROM listening_events WHERE subject_id = ? AND occurred_at < ?",
+  ).get(subjectId, before)?.occurred_at;
+  // A history that starts with this read has nothing to have missed.
+  if (!after) return;
+  database.prepare(
+    `INSERT OR IGNORE INTO listening_coverage_gaps (subject_id, source_key, gap_start, gap_end, detected_at)
+    VALUES (?, ?, ?, ?, ?)`,
+  ).run(subjectId, sourceKey, after, before, detectedAt);
+}
+
+// Recent syncs from before gaps were recorded: a run that stored a full window of 50
+// new plays at once may have missed plays before its oldest one.
+function recordEarlierRecentSyncGaps(database) {
+  const runs = database.prepare(
+    `SELECT subject_id, json_extract(record_json, '$.ingested_at') AS ingested_at,
+      MIN(occurred_at) AS earliest, COUNT(*) AS plays
+    FROM listening_events
+    WHERE json_extract(record_json, '$.provenance.external_ref.entity_type') = 'spotify.play_history'
+    GROUP BY subject_id, ingested_at HAVING plays >= 50`,
+  ).all();
+  for (const run of runs) {
+    recordCoverageGap(database, {
+      subjectId: run.subject_id, sourceKey: "spotify.recently_played", before: run.earliest, detectedAt: run.ingested_at,
+    });
+  }
+}
+
 function insertBundleRecords(bundle, statements) {
   let insertedTracks = 0;
   let insertedEvents = 0;
@@ -569,6 +636,165 @@ function insertBundleRecords(bundle, statements) {
   return { insertedTracks, insertedEvents };
 }
 
+// Where stored plays came from, in words the agent can repeat. Others keep their kind.
+const COVERAGE_SOURCE_LABELS = {
+  "spotify.play_history": "Spotify recent plays, read while Moondog runs",
+  "spotify.extended_history.play": "Spotify Extended Streaming History file",
+  "spotify.account_data.play_history": "Spotify Account Data file",
+  "listenbrainz.listen": "ListenBrainz export file",
+};
+
+const SCROBBLE_SOURCE_KEY = "lastfm.music_import";
+// Scrobbles record when a song started. Spotify records when a play ended, so its start is
+// the end minus the listening time; Account Data times are rounded to the minute.
+const MIRROR_TOLERANCE_MS = 150_000;
+const DAY_MS = 86_400_000;
+const foldLabel = (value) => String(value ?? "").normalize("NFKC").toLocaleLowerCase("und")
+  .replace(/[^\p{L}\p{N}]+/gu, " ").trim();
+const lexicalCompare = (left, right) => (left < right ? -1 : left > right ? 1 : 0);
+
+/**
+ * Many Last.fm scrobbles repeat plays that Spotify or ListenBrainz recorded too. Each scrobble
+ * that matches another service's play (same artist and title, starting within a few minutes)
+ * is linked to it once, whichever was imported first, so the play counts once.
+ */
+function reconcileScrobbleMirrors(database, subjectId, earliest, latest, createdAt) {
+  if (!subjectId || !earliest || !latest) return 0;
+  const scrobbled = database.prepare(
+    "SELECT 1 FROM import_batches WHERE subject_id = ? AND source_key = ? LIMIT 1",
+  ).get(subjectId, SCROBBLE_SOURCE_KEY);
+  if (!scrobbled) return 0;
+  const rows = database.prepare(
+    `SELECT e.listening_event_id AS id, e.occurred_at AS at,
+      e.record_json AS event_json, t.record_json AS track_json
+    FROM listening_events e
+    JOIN track_refs t
+      ON t.track_ref_id = e.track_ref_id AND t.revision = e.track_ref_revision
+    WHERE e.subject_id = ? AND e.occurred_at BETWEEN ? AND ?
+      AND NOT EXISTS (
+        SELECT 1 FROM listening_event_supersessions s
+        WHERE s.superseded_event_id = e.listening_event_id
+      )
+      AND NOT EXISTS (
+        SELECT 1 FROM listening_event_mirrors m
+        WHERE m.mirror_event_id = e.listening_event_id
+          OR m.original_event_id = e.listening_event_id
+      )`,
+  ).all(
+    subjectId,
+    new Date(Date.parse(earliest) - DAY_MS).toISOString(),
+    new Date(Date.parse(latest) + DAY_MS).toISOString(),
+  );
+  const scrobbles = new Map();
+  const plays = [];
+  for (const row of rows) {
+    const event = JSON.parse(row.event_json);
+    const track = JSON.parse(row.track_json);
+    const artist = foldLabel(track.artist_credits?.[0]?.name);
+    const title = foldLabel(track.title);
+    if (!artist || !title) continue;
+    const key = `${artist}\u0000${title}`;
+    const at = Date.parse(row.at);
+    const system = String(event.provenance?.external_ref?.system ?? "");
+    if (system === "lastfm") {
+      if (!scrobbles.has(key)) scrobbles.set(key, []);
+      scrobbles.get(key).push({ id: row.id, start: at });
+      continue;
+    }
+    const length = Number.isSafeInteger(event.played_ms) ? event.played_ms
+      : Number.isSafeInteger(track.duration_ms) ? track.duration_ms : null;
+    // A Spotify play of unknown length can't be placed precisely enough to match.
+    if (system.startsWith("spotify") && length === null) continue;
+    plays.push({ id: row.id, key, start: system.startsWith("spotify") ? at - length : at });
+  }
+  const pairs = [];
+  for (const list of scrobbles.values()) list.sort((left, right) => left.start - right.start);
+  for (const play of plays) {
+    const list = scrobbles.get(play.key);
+    if (!list) continue;
+    let low = 0;
+    let high = list.length;
+    while (low < high) {
+      const middle = (low + high) >> 1;
+      if (list[middle].start < play.start - MIRROR_TOLERANCE_MS) low = middle + 1;
+      else high = middle;
+    }
+    for (let index = low; index < list.length && list[index].start <= play.start + MIRROR_TOLERANCE_MS; index += 1) {
+      pairs.push({ gap: Math.abs(list[index].start - play.start), play: play.id, scrobble: list[index].id });
+    }
+  }
+  // Closest pairs first, so each play and each scrobble is used once.
+  pairs.sort((left, right) => left.gap - right.gap ||
+    lexicalCompare(left.scrobble, right.scrobble) || lexicalCompare(left.play, right.play));
+  const insert = database.prepare(
+    `INSERT INTO listening_event_mirrors (mirror_event_id, original_event_id, created_at)
+    VALUES (?, ?, ?)`,
+  );
+  const usedPlays = new Set();
+  const usedScrobbles = new Set();
+  let linked = 0;
+  for (const pair of pairs) {
+    if (usedPlays.has(pair.play) || usedScrobbles.has(pair.scrobble)) continue;
+    usedPlays.add(pair.play);
+    usedScrobbles.add(pair.scrobble);
+    linked += insert.run(pair.scrobble, pair.play, createdAt).changes;
+  }
+  return linked;
+}
+
+function preserveExtendedHistoryMatches(bundle, retainedMatch) {
+  const candidates = bundle.reconciliationCandidates;
+  if (bundle.importBatch.sourceFormat !== "spotify_extended_streaming_history_music_v1") return candidates;
+  const retained = new Map(candidates.map((candidate) => [
+    candidate.supersedingEventId, retainedMatch(candidate.supersedingEventId),
+  ]));
+  if (!candidates.some((candidate) => retained.get(candidate.supersedingEventId) &&
+    retained.get(candidate.supersedingEventId) !== candidate.supersededEventId)) return candidates;
+
+  // Account Data cannot distinguish equal-duration plays of the same title and
+  // artist within one minute. Added video plays can reorder those ordinals.
+  // A narrower export may omit an established counterpart's ordinal entirely.
+  // Keep those pairs too; only new plays use this archive's remaining candidates.
+  // The store's conflict checks still reject a new play claiming an absent play's
+  // retained counterpart, so an incomplete export cannot invent another match.
+  const events = new Map(bundle.listeningEvents.map((event) => [event.listening_event_id, event]));
+  const tracks = new Map(bundle.trackRefs.map((track) => [track.track_ref_id, track]));
+  const groups = new Map();
+  for (const candidate of candidates) {
+    const event = events.get(candidate.supersedingEventId);
+    const track = tracks.get(event.track_ref_id);
+    const key = JSON.stringify([
+      event.occurred_at.slice(0, 16), event.played_ms,
+      track.title.normalize("NFKC").toLowerCase(),
+      track.artist_credits[0].name.normalize("NFKC").toLowerCase(),
+    ]);
+    const group = groups.get(key) ?? [];
+    group.push(candidate);
+    groups.set(key, group);
+  }
+  const matches = new Map();
+  for (const group of groups.values()) {
+    const available = new Set(group.map((candidate) => candidate.supersededEventId));
+    for (const candidate of group) {
+      const previous = retained.get(candidate.supersedingEventId);
+      if (!previous) continue;
+      available.delete(previous);
+      matches.set(candidate.supersedingEventId, previous);
+    }
+    const remaining = available.values();
+    for (const candidate of group) {
+      if (matches.has(candidate.supersedingEventId)) continue;
+      const selected = available.has(candidate.supersededEventId)
+        ? candidate.supersededEventId : remaining.next().value;
+      matches.set(candidate.supersedingEventId, selected);
+      available.delete(selected);
+    }
+  }
+  return candidates.map((candidate) => ({
+    ...candidate, supersededEventId: matches.get(candidate.supersedingEventId),
+  }));
+}
+
 export class ListeningHistoryStore {
   #database;
   #closed = false;
@@ -670,6 +896,10 @@ export class ListeningHistoryStore {
           AND NOT EXISTS (
             SELECT 1 FROM listening_event_supersessions s
             WHERE s.superseded_event_id = e.listening_event_id
+          )
+          AND NOT EXISTS (
+            SELECT 1 FROM listening_event_mirrors m
+            WHERE m.mirror_event_id = e.listening_event_id
           )`,
       )
       .get(normalized).count;
@@ -1024,14 +1254,24 @@ export class ListeningHistoryStore {
     this.#database.exec("BEGIN IMMEDIATE");
     let insertedTracks;
     let insertedEvents;
+    let mirroredEvents = 0;
     try {
       if (subjectId !== undefined) {
         this.localSubjectId({ preferredSubjectId: subjectId });
       }
+      const times = bundle.listeningEvents.map((record) => record.occurred_at).sort();
+      // A read that overlaps a stored play joins up with it.
+      const overlaps = bundle.listeningEvents.some((record) => statements.hasEvent.get(record.deduplication_key));
+      if (bundle.windowFull && subjectId !== undefined && !overlaps) {
+        recordCoverageGap(this.#database, { subjectId, sourceKey: bundle.sourceKey, before: times[0], detectedAt: bundle.capturedAt });
+      }
       ({ insertedTracks, insertedEvents } = insertBundleRecords(
         bundle,
         statements,
       ));
+      // A read that finds nothing new still moves the source's checked time.
+      statements.touchCursor.run(bundle.capturedAt, bundle.sourceKey);
+      mirroredEvents = reconcileScrobbleMirrors(this.#database, subjectId, times[0], times.at(-1), bundle.capturedAt);
       this.#database.exec("COMMIT");
     } catch (error) {
       this.#database.exec("ROLLBACK");
@@ -1045,6 +1285,7 @@ export class ListeningHistoryStore {
       inserted_events: insertedEvents,
       duplicate_events: bundle.listeningEvents.length - insertedEvents,
       inserted_track_refs: insertedTracks,
+      ...(mirroredEvents ? { mirrored_events: mirroredEvents } : {}),
       cursor_after_ms: this.sourceCursor(bundle.sourceKey),
       profile_effects: insertedEvents > 0 ? "updated" : "unchanged",
     };
@@ -1055,7 +1296,7 @@ export class ListeningHistoryStore {
     const bundle = normalizedImportBundle(value);
     const batch = bundle.importBatch;
     const existingBatch = this.#database.prepare(
-      `SELECT import_batch_id FROM import_batches
+      `SELECT * FROM import_batches
       WHERE import_batch_id = ?`,
     );
     const insertBatch = this.#database.prepare(
@@ -1141,12 +1382,14 @@ export class ListeningHistoryStore {
     let insertedTracks = 0;
     let insertedEvents = 0;
     let supersededEvents = 0;
+    let mirroredEvents = 0;
     let insertedProfileEvidence = 0;
     let alreadyImported = false;
     let profileAlreadyImported = bundle.profileImport === undefined;
     try {
       this.localSubjectId({ preferredSubjectId: batch.subjectId });
-      alreadyImported = Boolean(existingBatch.get(batch.importBatchId));
+      const previousBatch = existingBatch.get(batch.importBatchId);
+      alreadyImported = Boolean(previousBatch);
       if (!alreadyImported) {
         ({ insertedTracks, insertedEvents } = insertBundleRecords(
           bundle,
@@ -1169,8 +1412,38 @@ export class ListeningHistoryStore {
           batch.inputRecords - insertedEvents,
           insertedTracks,
         );
+      } else {
+        // Earlier readers skipped video members, even when they contained music.
+        // Extend that same archive's receipt without replacing its existing rows.
+        const previousMembers = JSON.parse(previousBatch.member_names_json);
+        const addedMembers = batch.memberNames.filter((name) => !previousMembers.includes(name));
+        const expandsExtendedArchive = batch.sourceFormat === "spotify_extended_streaming_history_music_v1" &&
+          previousBatch.subject_id === batch.subjectId && previousBatch.source_key === bundle.sourceKey &&
+          previousBatch.source_format === batch.sourceFormat && previousBatch.data_scope === batch.dataScope &&
+          previousBatch.archive_sha256 === batch.archiveSha256 && previousBatch.archive_size_bytes === batch.archiveSizeBytes &&
+          previousMembers.every((name) => batch.memberNames.includes(name)) && addedMembers.length > 0 &&
+          addedMembers.every((name) => /^Streaming_History_Video_\d{4}(?:_\d+)?\.json$/u.test(name));
+        if (expandsExtendedArchive) {
+          const incomingIds = new Set(bundle.listeningEvents.map((event) => event.listening_event_id));
+          const previousEvents = this.#database.prepare(`SELECT listening_event_id FROM listening_events
+            WHERE subject_id = ? AND json_extract(record_json, '$.provenance.import_batch_id') = ?`)
+            .all(batch.subjectId, batch.importBatchId);
+          if (batch.inputRecords < previousBatch.input_records || previousEvents.some((event) => !incomingIds.has(event.listening_event_id))) {
+            throw new Error("Extended archive expansion must preserve every previously imported event");
+          }
+          ({ insertedTracks, insertedEvents } = insertBundleRecords(bundle, statements));
+          this.#database.prepare(`UPDATE import_batches SET member_names_json = ?, input_records = ?,
+            earliest_occurred_at = ?, latest_occurred_at = ?,
+            inserted_events = inserted_events + ?, duplicate_events = ? - inserted_events - ?,
+            inserted_track_refs = inserted_track_refs + ? WHERE import_batch_id = ?`)
+            .run(JSON.stringify(batch.memberNames), batch.inputRecords, batch.earliestOccurredAt, batch.latestOccurredAt,
+              insertedEvents, batch.inputRecords, insertedEvents, insertedTracks, batch.importBatchId);
+        }
       }
-      for (const candidate of bundle.reconciliationCandidates) {
+      const reconciliationCandidates = preserveExtendedHistoryMatches(bundle, (eventId) =>
+        selectCandidateBySuperseding.get(eventId)?.superseded_event_id ??
+        selectBySuperseding.get(eventId)?.superseded_event_id);
+      for (const candidate of reconciliationCandidates) {
         const existingBySuperseded = selectCandidateBySuperseded.get(
           candidate.supersededEventId,
         )?.superseding_event_id;
@@ -1292,6 +1565,9 @@ export class ListeningHistoryStore {
           );
         }
       }
+      mirroredEvents = reconcileScrobbleMirrors(
+        this.#database, batch.subjectId, batch.earliestOccurredAt, batch.latestOccurredAt, batch.importedAt,
+      );
       this.#database.exec("COMMIT");
     } catch (error) {
       this.#database.exec("ROLLBACK");
@@ -1310,7 +1586,8 @@ export class ListeningHistoryStore {
       duplicate_events: batch.inputRecords - insertedEvents,
       inserted_track_refs: insertedTracks,
       superseded_events: supersededEvents,
-      effective_event_delta: insertedEvents - supersededEvents,
+      ...(mirroredEvents ? { mirrored_events: mirroredEvents } : {}),
+      effective_event_delta: insertedEvents - supersededEvents - mirroredEvents,
       cursor_after_ms: this.sourceCursor(bundle.sourceKey),
       earliest_occurred_at: batch.earliestOccurredAt,
       latest_occurred_at: batch.latestOccurredAt,
@@ -1333,7 +1610,7 @@ export class ListeningHistoryStore {
           }
         : {}),
       profile_effects:
-        insertedEvents - supersededEvents !== 0 || insertedProfileEvidence > 0
+        insertedEvents > 0 || supersededEvents > 0 || insertedProfileEvidence > 0
           ? "updated"
           : "unchanged",
     };
@@ -1355,6 +1632,10 @@ export class ListeningHistoryStore {
             SELECT 1 FROM listening_event_supersessions
             WHERE superseded_event_id = listening_events.listening_event_id
           )
+          AND NOT EXISTS (
+            SELECT 1 FROM listening_event_mirrors m
+            WHERE m.mirror_event_id = listening_events.listening_event_id
+          )
         ORDER BY occurred_at DESC, listening_event_id DESC
         LIMIT ?`,
       )
@@ -1454,6 +1735,10 @@ export class ListeningHistoryStore {
             SELECT 1 FROM listening_event_supersessions s
             WHERE s.superseded_event_id = e.listening_event_id
           )
+          AND NOT EXISTS (
+            SELECT 1 FROM listening_event_mirrors m
+            WHERE m.mirror_event_id = e.listening_event_id
+          )
         ORDER BY e.occurred_at, e.listening_event_id`,
       )
       .all(normalizedSubjectId)
@@ -1517,6 +1802,10 @@ export class ListeningHistoryStore {
             SELECT 1 FROM listening_event_supersessions
             WHERE superseded_event_id = listening_events.listening_event_id
           )
+          AND NOT EXISTS (
+            SELECT 1 FROM listening_event_mirrors m
+            WHERE m.mirror_event_id = listening_events.listening_event_id
+          )
         GROUP BY track_ref_id
         ORDER BY play_count DESC, track_ref_id`,
       )
@@ -1545,12 +1834,131 @@ export class ListeningHistoryStore {
     return structuredClone({ ...value.summary, lyric_exclusions: value.lyricExclusions });
   }
 
+  activeAvoidances({ subjectId } = {}) {
+    // Host-side filtering must include corrections outside the bounded display.
+    return structuredClone(
+      this.#profileProjection({ subjectId, maxItems: 1 }).lyricExclusions,
+    );
+  }
+
   profileSummary({ subjectId, maxItems = 10 } = {}) {
     return structuredClone(
       this.#profileProjection({ subjectId, maxItems }).summary,
     );
   }
 
+  profileCatalog({ subjectId } = {}) {
+    const projected = this.#profileProjection({ subjectId, maxItems: 1 });
+    const normalizedSubjectId = subjectId.toLowerCase();
+    const imports = this.#database.prepare(`SELECT source_format AS format, data_scope AS scope,
+      SUM(input_records) AS input_records, MIN(earliest_occurred_at) AS earliest,
+      MAX(latest_occurred_at) AS latest
+      FROM import_batches WHERE subject_id = ? GROUP BY source_format, data_scope ORDER BY source_format`)
+      .all(normalizedSubjectId);
+    const stored = this.#database.prepare("SELECT COUNT(*) AS count FROM listening_events WHERE subject_id = ?")
+      .get(normalizedSubjectId).count;
+    return structuredClone({
+      coverage: { ...projected.summary.coverage,
+        stored_listening_events: stored,
+        superseded_listening_events: stored - projected.summary.coverage.effective_listening_events },
+      sources: imports,
+      context: {
+        reference_date: projected.summary.listening_behavior.context.reference_date,
+        recent_window_days: projected.summary.listening_behavior.context.recent_window_days,
+        incognito_events_excluded: projected.summary.listening_behavior.context.incognito_events_excluded,
+        release_minimum_distinct_tracks: projected.summary.listening_behavior.context.release_minimum_distinct_tracks,
+      },
+      sections: projected.catalog,
+    });
+  }
+
+  // What the listener played lately: stored plays in the last `days` up to `now`, by artist.
+  // It reads the clock, not the newest play, so a history that stopped shows as quiet.
+  recentAttention({ subjectId, days = 28, now = Date.now(), limit = 10 } = {}) {
+    if (this.#closed) throw new Error("Listening history store is closed");
+    if (!isUuid(subjectId)) throw new TypeError("A valid profile subject is required");
+    const from = new Date(now - days * DAY_MS).toISOString();
+    const effective = `e.subject_id = ? AND e.occurred_at >= ?
+      AND NOT EXISTS (SELECT 1 FROM listening_event_supersessions s WHERE s.superseded_event_id = e.listening_event_id)
+      AND NOT EXISTS (SELECT 1 FROM listening_event_mirrors m WHERE m.mirror_event_id = e.listening_event_id)`;
+    const subject = subjectId.toLowerCase();
+    const plays = this.#database.prepare(`SELECT COUNT(*) AS plays FROM listening_events e WHERE ${effective}`)
+      .get(subject, from).plays;
+    const artists = this.#database.prepare(`SELECT json_extract(a.value, '$.name') AS name, COUNT(*) AS plays,
+      MAX(e.occurred_at) AS last_played_at
+      FROM listening_events e
+      JOIN track_refs t ON t.track_ref_id = e.track_ref_id AND t.revision = e.track_ref_revision
+      JOIN json_each(t.record_json, '$.artist_credits') a
+      WHERE ${effective} AND name IS NOT NULL
+      GROUP BY name ORDER BY plays DESC, last_played_at DESC LIMIT ?`).all(subject, from, limit);
+    return structuredClone({ from, to: new Date(now).toISOString(), plays, top_artists: artists });
+  }
+
+  // Which spans of time the stored plays cover, and where plays may be missing.
+  listeningCoverage({ subjectId } = {}) {
+    if (this.#closed) throw new Error("Listening history store is closed");
+    if (!isUuid(subjectId)) throw new TypeError("A valid profile subject is required");
+    const subject = subjectId.toLowerCase();
+    const sources = this.#database.prepare(`SELECT json_extract(record_json, '$.provenance.external_ref.entity_type') AS kind,
+      COUNT(*) AS plays, MIN(occurred_at) AS first_played_at, MAX(occurred_at) AS last_played_at
+      FROM listening_events WHERE subject_id = ? GROUP BY kind ORDER BY first_played_at`).all(subject);
+    // A history file imported later can fill a span that a sync missed.
+    const filled = this.#database.prepare(`SELECT earliest_occurred_at AS earliest, latest_occurred_at AS latest
+      FROM import_batches WHERE subject_id = ? AND earliest_occurred_at IS NOT NULL AND latest_occurred_at IS NOT NULL`).all(subject);
+    const gaps = this.#database.prepare(`SELECT gap_start AS "from", gap_end AS "to"
+      FROM listening_coverage_gaps WHERE subject_id = ? ORDER BY gap_start`).all(subject)
+      .flatMap((gap) => filled.reduce((spans, { earliest, latest }) => spans.flatMap((span) =>
+        latest <= span.from || earliest >= span.to ? [span] : [
+          ...(earliest > span.from ? [{ from: span.from, to: earliest }] : []),
+          ...(latest < span.to ? [{ from: latest, to: span.to }] : []),
+        ]), [{ ...gap }]));
+    const checked = this.#database.prepare("SELECT updated_at FROM source_cursors WHERE source_key = 'spotify.recently_played'").get();
+    return structuredClone({
+      sources: sources.map(({ kind, ...source }) => ({ source: COVERAGE_SOURCE_LABELS[kind] ?? kind, ...source })),
+      gaps: gaps.slice(-10),
+      ...(checked ? { recent_plays_checked_at: checked.updated_at } : {}),
+    });
+  }
+
+  listenerProfileStorage({ subjectId } = {}) {
+    if (this.#closed) throw new Error("Listening history store is closed");
+    if (!isUuid(subjectId)) throw new TypeError("A valid profile subject is required");
+    return new ListenerProfileStore(this.#database, subjectId.toLowerCase());
+  }
+
+  profileDataVersion() {
+    if (this.#closed) throw new Error("Listening history store is closed");
+    // data_version observes other connections; total_changes observes our own.
+    // This is a connection-local cache key, never a persisted evidence digest.
+    return `${this.#database.prepare("PRAGMA data_version").get().data_version}:${
+      this.#database.prepare("SELECT total_changes() AS changes").get().changes}`;
+  }
+
+  profileEvidenceRevision({ subjectId } = {}) {
+    if (this.#closed) throw new Error("Listening history store is closed");
+    if (!isUuid(subjectId)) throw new TypeError("A valid profile subject is required");
+    const digest = createHash("sha256");
+    // Build artifacts never feed back into source evidence. Hash locally; raw
+    // events, source identifiers and private import metadata do not go to Pi.
+    const queries = [
+      "SELECT record_json FROM listening_events WHERE subject_id = ? ORDER BY listening_event_id",
+      "SELECT record_json FROM spotify_profile_evidence WHERE subject_id = ? ORDER BY profile_evidence_id",
+      "SELECT record_json FROM taste_events WHERE subject_id = ? ORDER BY taste_event_id",
+      "SELECT * FROM import_batches WHERE subject_id = ? ORDER BY import_batch_id",
+      "SELECT * FROM profile_import_batches WHERE subject_id = ? ORDER BY profile_import_id",
+      `SELECT t.record_json FROM track_refs t WHERE EXISTS (SELECT 1 FROM listening_events e
+        WHERE e.subject_id = ? AND e.track_ref_id = t.track_ref_id AND e.track_ref_revision = t.revision)
+        ORDER BY t.track_ref_id, t.revision`,
+    ];
+    for (const query of queries) {
+      digest.update(query).update("\0");
+      for (const row of this.#database.prepare(query).iterate(subjectId.toLowerCase())) {
+        digest.update(JSON.stringify(row)).update("\0");
+      }
+    }
+    return digest.digest("hex");
+  }
+
   rediscoveryCandidates({ subjectId, limit = 6 } = {}) {
     if (!Number.isInteger(limit) || limit < 1 || limit > 12) {
       throw new TypeError("Listening rediscovery limit is invalid");
@@ -1930,6 +2338,10 @@ export class ListeningHistoryStore {
           WHERE NOT EXISTS (
             SELECT 1 FROM listening_event_supersessions
             WHERE superseded_event_id = listening_events.listening_event_id
+          )
+          AND NOT EXISTS (
+            SELECT 1 FROM listening_event_mirrors m
+            WHERE m.mirror_event_id = listening_events.listening_event_id
           )`,
         )
         .get().count,
@@ -2003,7 +2415,11 @@ async function initializeListeningHistoryDatabase(
   ) {
     throw new Error("Listening history database schema is incompatible");
   }
+  const recordsGaps = database.prepare(
+    "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'listening_coverage_gaps'",
+  ).get();
   database.exec(schema);
+  if (!recordsGaps) recordEarlierRecentSyncGaps(database);
   if (currentVersion < LISTENING_HISTORY_SCHEMA_VERSION) {
     database.exec(
       `PRAGMA user_version = ${LISTENING_HISTORY_SCHEMA_VERSION}`,