@audiofool/moondog 0.2.0-beta.2 → 0.2.0-beta.4

package/src/integrations/spotify/catalog-resolver.mjs

@@ -1,3 +1,4 @@
+import { ARTIST_CREDIT_SEPARATORS as artistSeparators } from "../../core/listener-avoids.mjs";
 import { spotifyResolutionCacheKey } from "./resolution-cache.mjs";
 
 export const SPOTIFY_CATALOG_RESOLVER_LIMITS = Object.freeze({
@@ -10,8 +11,6 @@ export const SPOTIFY_CATALOG_RESOLVER_LIMITS = Object.freeze({
   durationToleranceMs: 3_000,
 });
 
-const artistSeparators =
-  /\s*(?:,|;|&|\/|(?:featuring|feat\.?|ft\.?|with)(?=\s|$)|、|和|与)\s*/iu;
 const bracketedDescriptorPattern = /[（(【\[]([^）)】\]]+)[）)】\]]/gu;
 const trailingDescriptorPattern = /^(.*)\s+(?:-|–)\s+(.+)$/u;
 const materialRecordingTags = new Set([

package/src/integrations/spotify/web-api-client.mjs

@@ -330,6 +330,9 @@ function normalizeTrackSearchItem(raw) {
   const popularity = safeInteger(raw.popularity, 0, 100);
   if (popularity !== undefined) item.popularity = popularity;
   if (typeof raw.explicit === "boolean") item.explicit = raw.explicit;
+  // Host-only, like the URI: it tells two copies of one recording apart from two songs.
+  const isrc = safeText(raw.external_ids?.isrc, 32);
+  if (isrc) item.isrc = isrc.toUpperCase();
   return item;
 }
 

package/src/memory/local-memory-store.mjs

@@ -572,8 +572,31 @@ function normalized(value) {
   return value.normalize("NFKC").toLocaleLowerCase("en-US");
 }
 
+// Memories the listener saved themselves: with remember, /remember or the desktop app, or by
+// asking Moondog in conversation, where the host checks the words against their message.
+const listenerSavedOrigins = ["explicit_user", "agent_from_explicit_user_statement"];
+
+// Text cut to `maximum` characters, ending in an ellipsis when it was cut.
+function clipped(value, maximum) {
+  const characters = Array.from(value);
+  return characters.length > maximum ? `${characters.slice(0, maximum).join("").trimEnd()}…` : value;
+}
+
+// Common English words that would recall unrelated memories on their own.
+const stopwords = new Set(`about after again all also am an and any are as at be been but by can could
+  did do does don for from get had has have he her here him his how if in into is it its just let like
+  me more most my no not now of on one or our out please really she so some than that the their them
+  then there these they this those to too up us very was we were what when where which who why will
+  with would you your`.split(/\s+/u));
+const cjkScript = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
+const cjkScripts = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu;
+// A word without a plural "s", so "record" and "records" find each other.
+const singular = (word) => (word.length > 3 && word.endsWith("s") && !word.endsWith("ss") ? word.slice(0, -1) : word);
+
+// Search terms in a query: Han bigrams, and whole words elsewhere. No query gives null, which
+// matches everything; a query of only common words gives no terms, which matches nothing.
 function queryTerms(value) {
-  if (typeof value !== "string") return [];
+  if (typeof value !== "string" || !value.trim()) return null;
   const text = normalized(value);
   const segments = text.match(/\p{Script=Han}+|[\p{L}\p{N}]+/gu) ?? [];
   const terms = [];
@@ -584,18 +607,21 @@ function queryTerms(value) {
       for (let index = 0; index < characters.length - 1; index += 1) {
         terms.push(`${characters[index]}${characters[index + 1]}`);
       }
-    } else if (Array.from(segment).length >= 2) {
-      terms.push(segment);
+    } else if (Array.from(segment).length >= 2 && !stopwords.has(segment)) {
+      terms.push(cjkScript.test(segment) ? segment : singular(segment));
     }
   }
   return [...new Set(terms)].slice(0, 24);
 }
 
+// CJK terms match inside the text, since those scripts do not space words; other terms match
+// whole words only, plural or not, so "me" no longer recalls "named".
 function relevanceScore(memory, terms) {
-  if (terms.length === 0) return 0;
+  if (!terms?.length) return 0;
   const content = normalized(memory.content);
+  const words = new Set((content.replace(cjkScripts, " ").match(/[\p{L}\p{N}]+/gu) ?? []).map(singular));
   return terms.reduce(
-    (score, term) => score + (content.includes(term) ? term.length : 0),
+    (score, term) => score + ((cjkScript.test(term) ? content.includes(term) : words.has(term)) ? term.length : 0),
     0,
   );
 }
@@ -724,6 +750,16 @@ export class LocalMemoryStore {
         timestamp,
         timestamp,
       );
+    // Saving again what the memory agent drew from talk makes it the listener's own.
+    if (inserted.changes === 0 && listenerSavedOrigins.includes(mutation.origin)) {
+      this.#database
+        .prepare(`
+          UPDATE memory_claims SET origin = ?
+          WHERE status = 'active' AND normalized_content = ?
+            AND origin NOT IN (${listenerSavedOrigins.map(() => "?").join(", ")})
+        `)
+        .run(mutation.origin, mutation.normalizedContent, ...listenerSavedOrigins);
+    }
     const row = this.#database
       .prepare(`
         SELECT *
@@ -1928,7 +1964,7 @@ export class LocalMemoryStore {
     const terms = queryTerms(query);
     const ranked = rows
       .map((row) => ({ row, score: relevanceScore(row, terms) }))
-      .filter(({ score }) => terms.length === 0 || score > 0)
+      .filter(({ score }) => terms === null || score > 0)
       .sort(
         (left, right) =>
           right.score - left.score ||
@@ -1978,7 +2014,7 @@ export class LocalMemoryStore {
         score: relevanceScore({ content: row.summary }, terms),
       }))
       .filter(
-        ({ score }) => terms.length === 0 || score > 0,
+        ({ score }) => terms === null || score > 0,
       )
       .sort(
         (left, right) =>
@@ -2030,29 +2066,50 @@ export class LocalMemoryStore {
     }));
   }
 
+  // What the agent reads on every model request, so it is kept short: no ids it cannot use,
+  // and earlier turns clipped.
   context({ query, currentSessionId } = {}) {
     this.#assertOpen();
-    const durableMemories = this.listMemories({ limit: 12 })
-      .filter((memory) => memory.kind !== "fact")
-      .slice(0, 6);
+    // Facts the memory agent drew from conversations come back only when a request is about
+    // them. A fact the listener saved themselves is something they want kept in mind, so it
+    // stays in view, in slots of its own so saved notes never push out a preference or a rule.
+    const latest = (filter, ...values) => this.#database.prepare(`SELECT * FROM memory_claims
+      WHERE status = 'active' AND ${filter} ORDER BY updated_at DESC, memory_id ASC LIMIT ?`).all(...values);
+    const durableMemories = [
+      ...latest("kind != 'fact'", 6),
+      ...latest(`kind = 'fact' AND origin IN (${listenerSavedOrigins.map(() => "?").join(", ")})`, ...listenerSavedOrigins, 4),
+    ].map(publicMemory);
     const durableIds = new Set(
       durableMemories.map((memory) => memory.memory_id),
     );
+    const memory = ({ memory_id, kind, horizon, text, created_at }) => ({ memory_id, kind, horizon, text, created_at });
     return {
-      durable_memories: durableMemories,
-      relevant_memories: this.listMemories({ limit: 8, query }).filter(
-        (memory) => !durableIds.has(memory.memory_id),
-      ),
+      durable_memories: durableMemories.map(memory),
+      relevant_memories: this.listMemories({ limit: 8, query })
+        .filter((candidate) => !durableIds.has(candidate.memory_id))
+        .map(memory),
       recent_episodes: this.recentEpisodes({
         query,
         excludeSessionId: currentSessionId,
         limit: 10,
-      }),
+      }).map((episode) => ({
+        kind: episode.kind,
+        summary: clipped(episode.summary, 400),
+        occurred_at: episode.occurred_at,
+        ...(episode.expires_at ? { expires_at: episode.expires_at } : {}),
+      })),
       recent_sessions: this.recentSessions({
         excludeSessionId: currentSessionId,
         limit: 2,
         turnsPerSession: 6,
-      }),
+      }).map((session) => ({
+        started_at: session.started_at,
+        ended_at: session.ended_at,
+        turns: session.turns.map((turn) => ({
+          ...turn,
+          text: clipped(turn.text, turn.role === "assistant" ? 600 : 400),
+        })),
+      })),
     };
   }
 

package/src/profile/listener-corrections.mjs

@@ -194,6 +194,28 @@ export function createListenerCorrectionRetraction({
   };
 }
 
+/**
+ * Records a Like or Avoid the way the profile command does, or retracts the active choice for
+ * the same artist or track. A retraction with no such choice changes nothing.
+ */
+export function recordListenerChoice(store, {
+  subjectId, entityType, label, artistCredit, stance, occurredAt = new Date().toISOString(),
+} = {}) {
+  if (stance !== "retract") {
+    return store.recordListenerCorrection({ subjectId, entityType, label, artistCredit, stance, occurredAt, recordedAt: occurredAt });
+  }
+  const key = (value) => typeof value === "string" ? normalizedIdentity(value.replace(/\s+/gu, " ").trim()) : "";
+  const active = store.listListenerCorrections({ subjectId, limit: 100 }).find((choice) =>
+    choice.entity_type === entityType && key(choice.label) === key(label) &&
+    (entityType !== "track" || key(choice.artist_credit) === key(artistCredit)));
+  if (!active) return { state: "not_found", entity_type: entityType, label };
+  return {
+    ...store.retractListenerCorrection({ subjectId, correctionId: active.correction_id, occurredAt, recordedAt: occurredAt }),
+    stance: active.stance,
+    ...(active.artist_credit ? { artist_credit: active.artist_credit } : {}),
+  };
+}
+
 export function listenerCorrectionTargetKey(record) {
   if (
     !record ||

package/src/profile/listener-profile-build.mjs

@@ -1,9 +1,10 @@
 import { createHash } from "node:crypto";
 
+import { avoidKey } from "../core/listener-avoids.mjs";
 import { recordedShare } from "./recorded-measures.mjs";
 
 export const LISTENER_PROFILE_ANALYSIS_VERSION = "listener-profile/2";
-export const LISTENER_PROFILE_SYNTHESIS_VERSION = "listener-synthesis/4";
+export const LISTENER_PROFILE_SYNTHESIS_VERSION = "listener-synthesis/5";
 // A saved reading is current only after its whole-profile check has run.
 export const CHECKED_PROFILE_STATES = new Set(["passed", "revised", "flagged"]);
 
@@ -74,6 +75,10 @@ function sectionSemantics(section, { time = "all" } = {}) {
       : "Not a preference ranking; listening time is not known for every play.",
     limitations: "Row order is not save or addition chronology. Missing listening minutes mean no matched duration in this import, not proof the listener never played a saved track.",
   };
+  if (section === "moondog_suggestions") return {
+    ordering: "Plays descending, then most recent.",
+    limitations: "Moondog queued or played these, so they are left out of every listening ranking and are not the listener's taste. adopted means the listener later played the song again on their own, which is a reaction to a suggestion, not a preference by itself.",
+  };
   if (section === "playlist_tracks") return { ...common,
     ordering: "Playlist membership count descending, then measured listening duration descending, then title." };
   if (section.startsWith("apple_")) return {
@@ -150,14 +155,16 @@ export function modelEvidence(entry, offset = 0, limit = 12) {
     "explicit_skips", "listening_minutes", "distinct_tracks", "year", "event_count", "first_observed_tracks",
     "playlist_count", "library_tracks", "preferred_tracks", "tracks_with_play_count", "preference_strength",
     "loved", "favorited", "rating_value", "rating_computed", "rank", "playlist_position", "value",
-    "stream_count", "played_seconds", "playlist_names", "top_artist", "first_year", "last_year", "active_years",
+    "stream_count", "played_seconds", "playlist_names", "top_artist", "first_year", "last_year", "active_years", "adopted",
+    "yearly_plays", "activation", "trajectory",
   ]);
   const data = Object.fromEntries(Object.entries(entry.data).filter(([key]) => fields.has(key)));
   if (data.top_artist) data.top_artist = Object.fromEntries(Object.entries(data.top_artist)
     .filter(([key]) => ["name", "play_count", "listening_minutes"].includes(key)));
   const continuations = {};
   for (const [key, value] of Object.entries(data)) {
-    if (Array.isArray(value)) {
+    // A yearly series is already capped, and a page of it would hide the latest years.
+    if (Array.isArray(value) && key !== "yearly_plays") {
       data[key] = value.slice(offset, offset + limit);
       if (value.length > offset + limit) continuations[key] = { total: value.length, next_offset: offset + limit };
     }
@@ -218,7 +225,16 @@ export function listenerProfileDossier(input) {
     sections,
     lifetime_artists: lifetime.slice(0, 40).map(row),
     next_lifetime_artists: lifetime.slice(40, 80).map(entry => ({ ref: entry.reference_id, name: entry.data.name,
-      listening_minutes: entry.data.listening_minutes, play_count: entry.data.play_count, distinct_tracks: entry.data.distinct_tracks })),
+      listening_minutes: entry.data.listening_minutes, play_count: entry.data.play_count, distinct_tracks: entry.data.distinct_tracks,
+      trajectory: entry.data.trajectory })),
+    movement: {
+      note: "Trajectory labels are computed from each artist's share of yearly plays: fading and rising compare the latest counted year with the years before it, returning means whole years without plays, or a decline, and then plays again, burst means one earlier year holds most of the plays and the next year drops off at once. They describe attention over time, not liking.",
+      ...Object.fromEntries(["fading", "returning", "rising", "burst"].map(label => [label, lifetime
+        .filter(entry => entry.data.trajectory === label).slice(0, 12)
+        .map(entry => ({ ref: entry.reference_id, name: entry.data.name, play_count: entry.data.play_count,
+          first_played_at: entry.data.first_played_at, last_played_at: entry.data.last_played_at,
+          yearly_plays: entry.data.yearly_plays }))])),
+    },
     recent_artists: rows("recent_artists").slice(0, 25).map(entry => ({ ...row(entry),
       lifetime_rank: lifetimeRank.get(artistKey(entry.data.name)) ?? null })),
     years: take("history_years", 64),
@@ -243,6 +259,13 @@ export function listenerProfileDossier(input) {
     },
     provider_signals: provider.slice(0, 50).map(row),
     explicit_choices: [...rows("listener_preferences"), ...rows("listener_avoids")].slice(0, 100).map(row),
+    ...(rows("moondog_suggestions").length ? { played_from_moondog: {
+      note: "Plays Moondog itself queued or played. They are not in any ranking above and are not taste. adopted means the listener played the song again later on their own.",
+      plays: input.manifest.context.moondog_started_events_excluded ?? null,
+      tracks: rows("moondog_suggestions").length,
+      adopted_tracks: rows("moondog_suggestions").filter(entry => entry.data.adopted === true).length,
+      examples: take("moondog_suggestions", 8),
+    } } : {}),
   };
 }
 
@@ -311,14 +334,62 @@ export function sameListenerChoices(evidence, otherEvidence) {
   return choices(evidence) === choices(otherEvidence);
 }
 
-export function readListenerProfile({ input, getInput, store }, { offset = 0, limit = 12, claimId, revisionId } = {}) {
+const CHOICES_SHOWN = 30;
+
+// A choice as the model reads it: no correction or evidence IDs.
+function choiceItems(evidence) {
+  return [...evidence.values()].filter(entry => LISTENER_SECTIONS.has(entry.section)).map(({ section, data }) => ({
+    key: [section, data.entity_type, avoidKey(data.label ?? data.name), avoidKey(data.artist_credit)].join("\0"),
+    choice: {
+      stance: section === "listener_avoids" ? "avoid" : "like",
+      subject: data.entity_type ?? "artist", name: data.label ?? data.name,
+      ...(data.artist_credit ? { artist: data.artist_credit } : {}),
+      ...(data.asserted_at ? { chosen_at: data.asserted_at } : {}),
+      ...(data.correction_id ? {} : { from_import: true }),
+    },
+  }));
+}
+
+// The listener's choices now, newest first, and how they differ from the ones a reading was built with.
+function listenerChoices(current, previous) {
+  const now = choiceItems(current);
+  const then = choiceItems(previous);
+  const direct = now.filter(item => !item.choice.from_import)
+    .sort((left, right) => (right.choice.chosen_at ?? "").localeCompare(left.choice.chosen_at ?? ""));
+  const missing = (items, others) => {
+    const keys = new Set(others.map(item => item.key));
+    return items.filter(item => !keys.has(item.key)).slice(0, CHOICES_SHOWN).map(item => item.choice);
+  };
+  return {
+    listener_choices: { total: direct.length, items: direct.slice(0, CHOICES_SHOWN).map(item => item.choice) },
+    choices_since_reading: { added: missing(now, then), retracted: missing(then, now) },
+  };
+}
+
+// The listener's current choices read straight from their saved corrections, newest first, for
+// when there is no saved reading to compare them with.
+export function savedListenerChoices(corrections, total) {
+  return {
+    total,
+    items: corrections.slice(0, CHOICES_SHOWN).map(choice => ({
+      stance: choice.stance, subject: choice.entity_type, name: choice.label,
+      ...(choice.artist_credit ? { artist: choice.artist_credit } : {}),
+      ...(choice.occurred_at ? { chosen_at: choice.occurred_at } : {}),
+    })),
+  };
+}
+
+export function readListenerProfile({ input, getInput, store, getChoices }, { offset = 0, limit = 12, claimId, revisionId } = {}) {
   if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isInteger(limit) || limit < 1 || limit > 20) {
     throw new TypeError("Saved profile page is invalid");
   }
   const current = store.current();
   const revision = revisionId ? store.revision(revisionId) : current;
   if (revisionId && !revision) throw new Error("That saved profile version is unavailable");
-  if (!revision) return { state: "missing", total_claims: 0, claims: [], next_offset: null };
+  if (!revision) {
+    const choices = getChoices?.();
+    return { state: "missing", total_claims: 0, claims: [], next_offset: null, ...(choices ? { listener_choices: choices } : {}) };
+  }
   // Most new listeners have no saved reading. Do not scan their source archive
   // merely to tell an ordinary conversation that the reading is missing.
   input ??= getInput();
@@ -345,6 +416,7 @@ export function readListenerProfile({ input, getInput, store }, { offset = 0, li
     }),
     listener_assertions: revision.claims.flatMap((claim, index) => claim.kind === "listener_assertion"
       ? [{ ...claim, finding_number: index + 1 }] : []),
+    ...listenerChoices(input.evidence ?? new Map(), new Map(Object.entries(revision.evidence ?? {}))),
   };
   if (claimId !== undefined) {
     const claim = revision.claims.find(item => item.claim_id === claimId);
@@ -357,22 +429,33 @@ export function readListenerProfile({ input, getInput, store }, { offset = 0, li
 }
 
 export function compactListenerProfile(value) {
-  if (!value || value.state === "missing") return { state: "missing" };
+  // With no reading, the listener's own choices still hold.
+  if (!value || value.state === "missing") {
+    return { state: "missing", ...(value?.listener_choices?.total ? { listener_choices: value.listener_choices } : {}) };
+  }
   const coverage = value.coverage?.coverage ?? {};
   const flagged = value.verification?.state === "flagged"
     ? value.verification.checks.at(-1).issues.slice(0, 3).map(issue => issue.problem) : [];
   const behind = value.state === "stale" && value.newer_listening_only === true;
+  // A stale reading still shows what it said, marked as the previous one, under the current choices.
+  const previous = value.state === "stale" && !behind && typeof value.summary === "string";
+  const since = value.choices_since_reading;
   return {
     state: behind ? "current_through" : value.state, revision_id: value.revision_id, sequence: value.sequence, built_at: value.built_at,
-    ...(value.state === "current" || behind ? {
+    ...(value.state === "current" || behind || previous ? {
       ...(behind ? { note: "Built from listening up to coverage.last_played_at. Plays stored since then are not in this reading; listening coverage shows them. Listener choices and analysis are unchanged." } : {}),
+      ...(previous ? { previous_reading: true,
+        note: "The previous reading. It predates listener_choices and choices_since_reading, or the analysis has changed since. Use it only as background: where they differ, the listener's choices win. Update the reading before treating it as current taste." } : {}),
       summary: value.summary,
       claims: value.highlights,
-      listener_assertions: value.listener_assertions,
+      // A previous reading's own choices may be out of date; listener_choices holds the current ones.
+      ...(previous ? {} : { listener_assertions: value.listener_assertions }),
       coverage: { listening_events: coverage.effective_listening_events ?? null,
         first_played_at: coverage.earliest_played_at ?? null, last_played_at: coverage.latest_played_at ?? null,
         library_tracks: coverage.apple_library_tracks ?? null },
       ...(flagged.length ? { unresolved_check_notes: flagged } : {}),
     } : { message: "Source evidence, listener choices or analysis changed. Update the reading before using the previous interpretation as current taste." }),
+    ...(value.listener_choices ? { listener_choices: value.listener_choices } : {}),
+    ...(since?.added?.length || since?.retracted?.length ? { choices_since_reading: since } : {}),
   };
 }

package/src/profile/listener-profile-worker.mjs

@@ -18,12 +18,19 @@ const HOUR_MS = 3_600_000;
 // or this one, holds off the next background attempt for half a day.
 export const PROFILE_REFRESH = { listeningAgeMs: 7 * 24 * HOUR_MS, newPlays: 300, attemptGapMs: 12 * HOUR_MS };
 
+const synthesisNumber = (version) => Number(/^listener-synthesis\/(\d+)$/u.exec(version ?? "")?.[1] ?? Number.NaN);
+
 /** Rebuild a saved reading that newer evidence has overtaken, without being asked. */
 export async function refreshListenerProfileIfDue({ application, now = Date.now(), ...options } = {}) {
   // The first reading is the listener's to ask for.
   if (!application.profileBuildReady()) return { state: "unavailable" };
   const reading = await application.getListenerProfile({ limit: 1 });
   if (reading.state !== "stale") return { state: reading.state };
+  // A reading a newer Moondog wrote is left to that version or to the listener's own build, so an
+  // older terminal and a newer app never take turns rebuilding over each other's reading.
+  if (synthesisNumber(reading.synthesis_version) > synthesisNumber(LISTENER_PROFILE_SYNTHESIS_VERSION)) {
+    return { state: "newer_version" };
+  }
   const { input, store } = application.getProfileBuildContext();
   const lastAttempt = Date.parse(store.lastAttemptAt() ?? "");
   if (now - lastAttempt < PROFILE_REFRESH.attemptGapMs) return { state: "waiting" };