@audiofool/moondog 0.2.0-beta.2 → 0.2.0-beta.3

package/src/profile/listening-history-store.mjs

@@ -9,10 +9,24 @@ import {
   createListenerCorrectionRetraction,
 } from "./listener-corrections.mjs";
 import { projectListeningProfile } from "./listening-profile-projection.mjs";
+import {
+  listeningFamiliarity, markMoondogStartedPlays, mixSlot, suggestionEntry, suggestionIdentities, SUGGESTION_WINDOW,
+} from "./moondog-suggestions.mjs";
 import { ListenerProfileStore, listenerProfileSchema } from "./listener-profile-store.mjs";
 
 export const LISTENING_HISTORY_SCHEMA_VERSION = 7;
 
+// The tables a cached projection, build input, Avoid gate or status reads, apart from the
+// suggestions ledger (listeningDataVersion). Rows are only added to the first ones: the one
+// update to a stored play, which records the play that supersedes it, is made with its row in
+// listening_event_supersessions. Imports update their batch rows, so those are read in full.
+// A new table those caches read, or a new update or delete, belongs here.
+const APPENDED_EVIDENCE_TABLES = Object.freeze([
+  "track_refs", "listening_events", "listening_event_supersessions", "listening_event_mirrors",
+  "spotify_profile_evidence", "taste_events",
+]);
+const UPDATED_EVIDENCE_TABLES = Object.freeze(["import_batches", "profile_import_batches"]);
+
 export function resolveListeningHistoryPath(environment = process.env) {
   return path.join(resolveMoondogStateDirectory(environment), "listening-history.sqlite");
 }
@@ -186,6 +200,26 @@ const schema = `
     ON taste_events(retracts_taste_event_id)
     WHERE retracts_taste_event_id IS NOT NULL;
 
+  -- What Moondog itself queued or played, written from the host's accepted receipts. Plays
+  -- that match one are left out of the listener's own attention when the profile is read
+  -- (src/profile/moondog-suggestions.mjs). Provider track identities never leave the host.
+  CREATE TABLE IF NOT EXISTS moondog_suggestions (
+    suggestion_id TEXT PRIMARY KEY NOT NULL,
+    service TEXT NOT NULL,
+    action TEXT NOT NULL,
+    provider_track_id TEXT,
+    isrc TEXT,
+    track_key TEXT,
+    release_key TEXT,
+    title TEXT NOT NULL,
+    artist_credit TEXT NOT NULL,
+    requested_at TEXT NOT NULL,
+    mix_slot TEXT
+  ) STRICT;
+
+  CREATE INDEX IF NOT EXISTS moondog_suggestions_time
+    ON moondog_suggestions(requested_at);
+
   ${listenerProfileSchema}
 `;
 
@@ -649,6 +683,13 @@ const SCROBBLE_SOURCE_KEY = "lastfm.music_import";
 // the end minus the listening time; Account Data times are rounded to the minute.
 const MIRROR_TOLERANCE_MS = 150_000;
 const DAY_MS = 86_400_000;
+
+// A local calendar date, such as 2026-10-25.
+function localDate(date) {
+  const pad = (value) => String(value).padStart(2, "0");
+  return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`;
+}
+
 const foldLabel = (value) => String(value ?? "").normalize("NFKC").toLocaleLowerCase("und")
   .replace(/[^\p{L}\p{N}]+/gu, " ").trim();
 const lexicalCompare = (left, right) => (left < right ? -1 : left > right ? 1 : 0);
@@ -798,6 +839,11 @@ function preserveExtendedHistoryMatches(bundle, retainedMatch) {
 export class ListeningHistoryStore {
   #database;
   #closed = false;
+  #snapshot = null;
+  #projection = null;
+  #coverageSources = null;
+  #evidence = null;
+  #ledgerChanges = 0;
 
   constructor(database) {
     this.#database = database;
@@ -1239,6 +1285,22 @@ export class ListeningHistoryStore {
       });
   }
 
+  // How many choices are in force, beyond the ones a list shows.
+  countActiveListenerCorrections({ subjectId } = {}) {
+    if (this.#closed) throw new Error("Listening history store is closed");
+    if (!isUuid(subjectId)) {
+      throw new TypeError("A valid listener correction subject is required");
+    }
+    return this.#database
+      .prepare(
+        `SELECT COUNT(*) AS count FROM taste_events a
+        WHERE a.subject_id = ? AND a.operation = 'assert'
+          AND NOT EXISTS (SELECT 1 FROM taste_events r WHERE r.retracts_taste_event_id = a.taste_event_id)
+          AND NOT EXISTS (SELECT 1 FROM taste_events s WHERE s.supersedes_taste_event_id = a.taste_event_id)`,
+      )
+      .get(subjectId.toLowerCase()).count;
+  }
+
   ingest(value) {
     if (this.#closed) throw new Error("Listening history store is closed");
     const bundle = normalizedBundle(value);
@@ -1772,15 +1834,103 @@ export class ListeningHistoryStore {
       )
       .all(normalizedSubjectId)
       .map((row) => JSON.parse(row.record_json));
-    return projectListeningProfile({
-      subjectId: normalizedSubjectId,
-      eventRows,
-      profileEvidence,
-      tasteEvents,
-      maxItems,
+    const { rows, adopted } = markMoondogStartedPlays(eventRows, this.#suggestionsForMatching());
+    return {
+      ...projectListeningProfile({
+        subjectId: normalizedSubjectId,
+        eventRows: rows,
+        profileEvidence,
+        tasteEvents,
+        maxItems,
+        adoptedSuggestionIds: adopted,
+      }),
+      adoptedSuggestionIds: adopted,
+    };
+  }
+
+  #suggestionsForMatching() {
+    return this.#database.prepare(`SELECT suggestion_id, provider_track_id, isrc, track_key, release_key, title, artist_credit,
+      requested_at FROM moondog_suggestions ORDER BY requested_at, suggestion_id`).all();
+  }
+
+  /** Records what Moondog itself queued or played. Only accepted host writes belong here. */
+  recordMoondogSuggestions({ entries } = {}) {
+    if (this.#closed) throw new Error("Listening history store is closed");
+    if (!Array.isArray(entries) || entries.length > 50) throw new TypeError("Moondog suggestions are invalid");
+    const insert = this.#database.prepare(`INSERT INTO moondog_suggestions
+      (suggestion_id, service, action, provider_track_id, isrc, track_key, release_key, title, artist_credit, requested_at, mix_slot)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
+    let recorded = 0;
+    this.#database.exec("BEGIN IMMEDIATE");
+    try {
+      for (const value of entries) {
+        const entry = suggestionEntry(value);
+        if (!entry || !/^[a-z][a-z0-9_.]{1,63}$/u.test(entry.action ?? "") || !Number.isFinite(Date.parse(entry.requested_at))) continue;
+        insert.run(randomUUID(), entry.service, entry.action, entry.provider_track_id, entry.isrc, entry.track_key, entry.release_key,
+          entry.title, entry.artist_credit, new Date(entry.requested_at).toISOString(), entry.mix_slot);
+        recorded += 1;
+      }
+      this.#database.exec("COMMIT");
+    } catch (error) {
+      this.#database.exec("ROLLBACK");
+      throw error;
+    }
+    this.#ledgerChanges += recorded;
+    return { recorded };
+  }
+
+  /**
+   * Moondog's earlier suggestions of each track, matched as stored plays are: the same provider
+   * track, ISRC or song. Adopted means the listener later played it again on their own.
+   */
+  moondogSuggestionHistory({ subjectId, tracks } = {}) {
+    if (!Array.isArray(tracks)) throw new TypeError("Suggestion history tracks are invalid");
+    const suggestions = this.#suggestionsForMatching();
+    if (!suggestions.length) return tracks.map(() => null);
+    const { adopted } = this.#listeningSnapshot(subjectId);
+    const byIdentity = new Map();
+    for (const suggestion of suggestions) {
+      for (const identity of suggestionIdentities(suggestion)) {
+        byIdentity.set(identity, [...(byIdentity.get(identity) ?? []), suggestion]);
+      }
+    }
+    return tracks.map((track) => {
+      const entry = suggestionEntry({ service: "spotify", action: "lookup", ...track });
+      const matches = new Set(entry ? suggestionIdentities(entry).flatMap((identity) => byIdentity.get(identity) ?? []) : []);
+      if (!matches.size) return null;
+      const last = [...matches].reduce((latest, item) => item.requested_at > latest ? item.requested_at : latest, "");
+      return { times: matches.size, last_suggested_at: last.slice(0, 10),
+        adopted: [...matches].some((item) => adopted.has(item.suggestion_id)) };
     });
   }
 
+  /** Where each track sits in the listener's own plays (MIX_SLOTS in moondog-suggestions.mjs). */
+  listeningMix({ subjectId, tracks } = {}) {
+    if (!Array.isArray(tracks)) throw new TypeError("Listening mix tracks are invalid");
+    const { familiarity } = this.#listeningSnapshot(subjectId);
+    return tracks.map((track) => mixSlot(track, familiarity));
+  }
+
+  // One projection with a single item per list serves the status, the Avoids, the catalog, the
+  // mix and the suggestion history until the listening data changes. Callers copy what they return.
+  #sharedProjection(subjectId) {
+    const version = `${subjectId}:${this.listeningDataVersion()}`;
+    if (this.#projection?.version !== version) {
+      this.#projection = { version, value: this.#profileProjection({ subjectId, maxItems: 1 }) };
+    }
+    return this.#projection.value;
+  }
+
+  #listeningSnapshot(subjectId) {
+    const version = `${subjectId}:${this.listeningDataVersion()}`;
+    if (this.#snapshot?.version !== version) {
+      const projected = this.#sharedProjection(subjectId);
+      this.#snapshot = { version, adopted: projected.adoptedSuggestionIds,
+        familiarity: listeningFamiliarity(projected.catalog, projected.summary.listening_behavior.context.reference_date) };
+    }
+    return this.#snapshot;
+  }
+
   /** Every track the subject has played, with its play count, most played first. */
   listenedTracks({ subjectId } = {}) {
     if (this.#closed) throw new Error("Listening history store is closed");
@@ -1836,19 +1986,16 @@ export class ListeningHistoryStore {
 
   activeAvoidances({ subjectId } = {}) {
     // Host-side filtering must include corrections outside the bounded display.
-    return structuredClone(
-      this.#profileProjection({ subjectId, maxItems: 1 }).lyricExclusions,
-    );
+    return structuredClone(this.#sharedProjection(subjectId).lyricExclusions);
   }
 
   profileSummary({ subjectId, maxItems = 10 } = {}) {
-    return structuredClone(
-      this.#profileProjection({ subjectId, maxItems }).summary,
-    );
+    return structuredClone((maxItems === 1 ? this.#sharedProjection(subjectId)
+      : this.#profileProjection({ subjectId, maxItems })).summary);
   }
 
   profileCatalog({ subjectId } = {}) {
-    const projected = this.#profileProjection({ subjectId, maxItems: 1 });
+    const projected = this.#sharedProjection(subjectId);
     const normalizedSubjectId = subjectId.toLowerCase();
     const imports = this.#database.prepare(`SELECT source_format AS format, data_scope AS scope,
       SUM(input_records) AS input_records, MIN(earliest_occurred_at) AS earliest,
@@ -1866,32 +2013,49 @@ export class ListeningHistoryStore {
         reference_date: projected.summary.listening_behavior.context.reference_date,
         recent_window_days: projected.summary.listening_behavior.context.recent_window_days,
         incognito_events_excluded: projected.summary.listening_behavior.context.incognito_events_excluded,
+        ...(projected.summary.listening_behavior.context.moondog_started_events_excluded
+          ? { moondog_started_events_excluded: projected.summary.listening_behavior.context.moondog_started_events_excluded } : {}),
         release_minimum_distinct_tracks: projected.summary.listening_behavior.context.release_minimum_distinct_tracks,
       },
       sections: projected.catalog,
     });
   }
 
-  // What the listener played lately: stored plays in the last `days` up to `now`, by artist.
-  // It reads the clock, not the newest play, so a history that stopped shows as quiet.
+  // What the listener played lately: stored plays on today and the `days` - 1 local days before
+  // it, by artist. It reads the clock, not the newest play, so a history that stopped shows as
+  // quiet. Whole local days keep the window, and so the model's context, the same all day.
   recentAttention({ subjectId, days = 28, now = Date.now(), limit = 10 } = {}) {
     if (this.#closed) throw new Error("Listening history store is closed");
     if (!isUuid(subjectId)) throw new TypeError("A valid profile subject is required");
-    const from = new Date(now - days * DAY_MS).toISOString();
+    const today = new Date(now);
+    const start = new Date(today);
+    start.setHours(0, 0, 0, 0);
+    start.setDate(start.getDate() - (days - 1));
+    const from = start.toISOString();
     const effective = `e.subject_id = ? AND e.occurred_at >= ?
       AND NOT EXISTS (SELECT 1 FROM listening_event_supersessions s WHERE s.superseded_event_id = e.listening_event_id)
       AND NOT EXISTS (SELECT 1 FROM listening_event_mirrors m WHERE m.mirror_event_id = e.listening_event_id)`;
     const subject = subjectId.toLowerCase();
-    const plays = this.#database.prepare(`SELECT COUNT(*) AS plays FROM listening_events e WHERE ${effective}`)
-      .get(subject, from).plays;
+    // Plays Moondog started are not the listener's own attention.
+    const suggestions = this.#suggestionsForMatching()
+      .filter((suggestion) => Date.parse(suggestion.requested_at) + SUGGESTION_WINDOW.afterMs >= Date.parse(from));
+    const started = suggestions.length === 0 ? [] : markMoondogStartedPlays(this.#database.prepare(`SELECT
+      e.listening_event_id, e.record_json AS event_json, t.record_json AS track_json FROM listening_events e
+      JOIN track_refs t ON t.track_ref_id = e.track_ref_id AND t.revision = e.track_ref_revision WHERE ${effective}`)
+      .all(subject, from).map((row) => ({ id: row.listening_event_id, event: JSON.parse(row.event_json), track: JSON.parse(row.track_json) })),
+    suggestions).rows.filter((row) => row.moondog_suggestion_id).map((row) => row.id);
+    const own = `${effective} AND e.listening_event_id NOT IN (SELECT value FROM json_each(?))`;
+    const plays = this.#database.prepare(`SELECT COUNT(*) AS plays FROM listening_events e WHERE ${own}`)
+      .get(subject, from, JSON.stringify(started)).plays;
     const artists = this.#database.prepare(`SELECT json_extract(a.value, '$.name') AS name, COUNT(*) AS plays,
       MAX(e.occurred_at) AS last_played_at
       FROM listening_events e
       JOIN track_refs t ON t.track_ref_id = e.track_ref_id AND t.revision = e.track_ref_revision
       JOIN json_each(t.record_json, '$.artist_credits') a
-      WHERE ${effective} AND name IS NOT NULL
-      GROUP BY name ORDER BY plays DESC, last_played_at DESC LIMIT ?`).all(subject, from, limit);
-    return structuredClone({ from, to: new Date(now).toISOString(), plays, top_artists: artists });
+      WHERE ${own} AND name IS NOT NULL
+      GROUP BY name ORDER BY plays DESC, last_played_at DESC LIMIT ?`).all(subject, from, JSON.stringify(started), limit);
+    return structuredClone({ from_date: localDate(start), through_date: localDate(today), plays,
+      ...(started.length ? { moondog_started_plays: started.length } : {}), top_artists: artists });
   }
 
   // Which spans of time the stored plays cover, and where plays may be missing.
@@ -1899,9 +2063,15 @@ export class ListeningHistoryStore {
     if (this.#closed) throw new Error("Listening history store is closed");
     if (!isUuid(subjectId)) throw new TypeError("A valid profile subject is required");
     const subject = subjectId.toLowerCase();
-    const sources = this.#database.prepare(`SELECT json_extract(record_json, '$.provenance.external_ref.entity_type') AS kind,
-      COUNT(*) AS plays, MIN(occurred_at) AS first_played_at, MAX(occurred_at) AS last_played_at
-      FROM listening_events WHERE subject_id = ? GROUP BY kind ORDER BY first_played_at`).all(subject);
+    // Reading every play is the slow part, so it is kept until the listening data changes.
+    const version = `${subject}:${this.listeningDataVersion()}`;
+    if (this.#coverageSources?.version !== version) {
+      this.#coverageSources = { version, sources: this.#database.prepare(`SELECT
+        json_extract(record_json, '$.provenance.external_ref.entity_type') AS kind,
+        COUNT(*) AS plays, MIN(occurred_at) AS first_played_at, MAX(occurred_at) AS last_played_at
+        FROM listening_events WHERE subject_id = ? GROUP BY kind ORDER BY first_played_at`).all(subject) };
+    }
+    const { sources } = this.#coverageSources;
     // A history file imported later can fill a span that a sync missed.
     const filled = this.#database.prepare(`SELECT earliest_occurred_at AS earliest, latest_occurred_at AS latest
       FROM import_batches WHERE subject_id = ? AND earliest_occurred_at IS NOT NULL AND latest_occurred_at IS NOT NULL`).all(subject);
@@ -1934,6 +2104,33 @@ export class ListeningHistoryStore {
       this.#database.prepare("SELECT total_changes() AS changes").get().changes}`;
   }
 
+  // A version of the listening evidence that no suggestions ledger write changes, from this
+  // connection or another, such as the other surface's. A new suggestion changes no projection
+  // until its plays arrive, so caches of the projection, the Avoids, the status and the build
+  // input key on this and are not rebuilt after every queue. This connection's own writes count
+  // by total_changes. Another connection's commit moves data_version whatever it wrote, so then
+  // the evidence tables themselves say whether the evidence changed.
+  listeningDataVersion() {
+    if (this.#closed) throw new Error("Listening history store is closed");
+    const dataVersion = this.#database.prepare("PRAGMA data_version").get().data_version;
+    const ownChanges = this.#database.prepare("SELECT total_changes() AS changes").get().changes - this.#ledgerChanges;
+    if (this.#evidence?.dataVersion !== dataVersion || this.#evidence.ownChanges !== ownChanges) {
+      this.#evidence = { dataVersion, ownChanges, fingerprint: this.#evidenceFingerprint() };
+    }
+    return this.#evidence.fingerprint;
+  }
+
+  #evidenceFingerprint() {
+    const digest = createHash("sha256");
+    for (const table of APPENDED_EVIDENCE_TABLES) {
+      digest.update(JSON.stringify(this.#database.prepare(`SELECT COUNT(*) AS rows, MAX(rowid) AS last FROM ${table}`).get()));
+    }
+    for (const table of UPDATED_EVIDENCE_TABLES) {
+      for (const row of this.#database.prepare(`SELECT * FROM ${table} ORDER BY rowid`).iterate()) digest.update(JSON.stringify(row));
+    }
+    return digest.digest("hex");
+  }
+
   profileEvidenceRevision({ subjectId } = {}) {
     if (this.#closed) throw new Error("Listening history store is closed");
     if (!isUuid(subjectId)) throw new TypeError("A valid profile subject is required");
@@ -1956,6 +2153,16 @@ export class ListeningHistoryStore {
         digest.update(JSON.stringify(row)).update("\0");
       }
     }
+    // Moondog's suggestions decide which plays count as the listener's own. A play can match a
+    // suggestion made up to SUGGESTION_WINDOW.beforeMs after it, for clock skew; one made later
+    // than that after the latest stored play matches nothing yet, so it leaves the reading current.
+    const latest = this.#database.prepare("SELECT MAX(occurred_at) AS latest FROM listening_events WHERE subject_id = ?")
+      .get(subjectId.toLowerCase()).latest;
+    const latestMs = Date.parse(latest ?? "");
+    const suggestions = Number.isFinite(latestMs) ? this.#database.prepare(`SELECT suggestion_id, provider_track_id, isrc, track_key,
+      release_key, title, artist_credit, requested_at FROM moondog_suggestions WHERE requested_at <= ? ORDER BY suggestion_id`)
+      .all(new Date(latestMs + SUGGESTION_WINDOW.beforeMs).toISOString()) : [];
+    if (suggestions.length) digest.update("moondog_suggestions\0").update(JSON.stringify(suggestions));
     return digest.digest("hex");
   }