@audiofool/moondog 0.1.0 → 0.2.0-beta.3

package/src/profile/listener-profile-store.mjs

@@ -0,0 +1,130 @@
+import { randomUUID } from "node:crypto";
+import { profileDigest } from "./listener-profile-build.mjs";
+
+// listener_profile_partitions holds page findings from builds before
+// listener-synthesis/4. New builds no longer write it; it stays so existing
+// databases open unchanged.
+export const listenerProfileSchema = `
+  CREATE TABLE IF NOT EXISTS listener_profile_builds (
+    build_id TEXT PRIMARY KEY NOT NULL, subject_id TEXT NOT NULL,
+    input_digest TEXT NOT NULL, model_key TEXT NOT NULL,
+    lease_token TEXT NOT NULL, state TEXT NOT NULL, record_json TEXT NOT NULL
+  ) STRICT;
+  CREATE INDEX IF NOT EXISTS listener_profile_build_input
+    ON listener_profile_builds(subject_id, input_digest, model_key);
+  CREATE TABLE IF NOT EXISTS listener_profile_partitions (
+    build_id TEXT NOT NULL, partition_id TEXT NOT NULL,
+    review_json TEXT NOT NULL,
+    PRIMARY KEY (build_id, partition_id),
+    FOREIGN KEY (build_id) REFERENCES listener_profile_builds(build_id)
+  ) STRICT;
+  CREATE INDEX IF NOT EXISTS listener_profile_partition_cache ON listener_profile_partitions(partition_id);
+  CREATE TABLE IF NOT EXISTS listener_profile_revisions (
+    revision_id TEXT PRIMARY KEY NOT NULL, subject_id TEXT NOT NULL,
+    sequence INTEGER NOT NULL, input_digest TEXT NOT NULL, record_json TEXT NOT NULL,
+    UNIQUE (subject_id, sequence)
+  ) STRICT;
+`;
+
+export class ListenerProfileStore {
+  constructor(database, subjectId) {
+    this.database = database;
+    this.subjectId = subjectId;
+  }
+
+  current() {
+    const row = this.database.prepare(`SELECT record_json FROM listener_profile_revisions
+      WHERE subject_id = ? ORDER BY sequence DESC LIMIT 1`).get(this.subjectId);
+    return row ? JSON.parse(row.record_json) : null;
+  }
+
+  revision(id) {
+    const row = this.database.prepare(`SELECT record_json FROM listener_profile_revisions
+      WHERE subject_id = ? AND revision_id = ?`).get(this.subjectId, id);
+    return row ? JSON.parse(row.record_json) : null;
+  }
+
+  /** When a build last started or resumed, whether the listener asked for it or not. */
+  lastAttemptAt() {
+    return this.database.prepare(`SELECT MAX(COALESCE(json_extract(record_json, '$.attempted_at'),
+      json_extract(record_json, '$.started_at'))) AS attempted_at
+      FROM listener_profile_builds WHERE subject_id = ?`).get(this.subjectId)?.attempted_at ?? null;
+  }
+
+  /** Resume an unfinished build for the same input and model, or start a new one. */
+  begin(input, model, { force = false } = {}) {
+    const modelKey = profileDigest(model);
+    this.database.exec("BEGIN IMMEDIATE");
+    try {
+      const parent = this.current()?.revision_id ?? null;
+      const existing = force ? null : this.database.prepare(`SELECT record_json FROM listener_profile_builds
+        WHERE subject_id = ? AND input_digest = ? AND model_key = ? AND state <> 'complete'
+        ORDER BY rowid DESC LIMIT 1`).get(this.subjectId, input.input_digest, modelKey);
+      const saved = existing ? JSON.parse(existing.record_json) : null;
+      // A checkpoint written against an older current reading cannot replace the newer one.
+      const resumable = saved?.parent_revision_id === parent ? saved : null;
+      const build = resumable ?? {
+        build_id: randomUUID(), input_digest: input.input_digest, model,
+        parent_revision_id: parent, started_at: new Date().toISOString(),
+      };
+      build.lease_token = randomUUID();
+      build.attempted_at = new Date().toISOString();
+      this.database.prepare(`INSERT INTO listener_profile_builds
+        (build_id, subject_id, input_digest, model_key, lease_token, state, record_json)
+        VALUES (?, ?, ?, ?, ?, 'building', ?) ON CONFLICT(build_id) DO UPDATE
+        SET lease_token = excluded.lease_token, state = 'building', record_json = excluded.record_json`)
+        .run(build.build_id, this.subjectId, input.input_digest, modelKey, build.lease_token, JSON.stringify(build));
+      this.database.exec("COMMIT");
+      return { ...build, resumed: Boolean(resumable) };
+    } catch (error) { this.database.exec("ROLLBACK"); throw error; }
+  }
+
+  #assertLease(build) {
+    const row = this.database.prepare(`SELECT lease_token, state FROM listener_profile_builds
+      WHERE subject_id = ? AND build_id = ?`).get(this.subjectId, build.build_id);
+    if (row?.lease_token !== build.lease_token || row.state !== "building") {
+      throw new Error("Another profile build has taken over; continue from its saved progress");
+    }
+  }
+
+  checkpointFinalization(build, finalization) {
+    this.database.exec("BEGIN IMMEDIATE");
+    try {
+      this.#assertLease(build);
+      const row = this.database.prepare("SELECT record_json FROM listener_profile_builds WHERE build_id = ?")
+        .get(build.build_id);
+      const record = { ...JSON.parse(row.record_json), finalization };
+      this.database.prepare("UPDATE listener_profile_builds SET record_json = ? WHERE build_id = ?")
+        .run(JSON.stringify(record), build.build_id);
+      this.database.exec("COMMIT");
+    } catch (error) { this.database.exec("ROLLBACK"); throw error; }
+  }
+
+  interrupt(build) {
+    this.database.prepare(`UPDATE listener_profile_builds SET state = 'interrupted'
+      WHERE subject_id = ? AND build_id = ? AND lease_token = ? AND state = 'building'`)
+      .run(this.subjectId, build.build_id, build.lease_token);
+  }
+
+  complete(build, profile, assertCurrentInput = () => {}) {
+    this.database.exec("BEGIN IMMEDIATE");
+    try {
+      this.#assertLease(build);
+      assertCurrentInput();
+      const previous = this.current();
+      if ((previous?.revision_id ?? null) !== build.parent_revision_id) {
+        throw new Error("The current profile changed during this build; start again from the new version");
+      }
+      const revision = {
+        ...profile, revision_id: randomUUID(), sequence: (previous?.sequence ?? 0) + 1,
+        parent_revision_id: previous?.revision_id ?? null, built_at: new Date().toISOString(), model: build.model,
+      };
+      this.database.prepare(`INSERT INTO listener_profile_revisions
+        (revision_id, subject_id, sequence, input_digest, record_json) VALUES (?, ?, ?, ?, ?)`)
+        .run(revision.revision_id, this.subjectId, revision.sequence, revision.input_digest, JSON.stringify(revision));
+      this.database.prepare("UPDATE listener_profile_builds SET state = 'complete' WHERE build_id = ?").run(build.build_id);
+      this.database.exec("COMMIT");
+      return revision;
+    } catch (error) { this.database.exec("ROLLBACK"); throw error; }
+  }
+}

package/src/profile/listener-profile-verification.mjs

@@ -0,0 +1,68 @@
+import { modelEvidence, profileDigest } from "./listener-profile-build.mjs";
+
+export const LISTENER_PROFILE_CHECK_VERSION = "listener-check/1";
+const MAX_CHECK_ISSUES = 6;
+
+export function listenerProfileCandidate(input, profile) {
+  const byId = new Map(profile.claims.map(claim => [claim.claim_id, claim]));
+  const insights = profile.highlight_claim_ids.map(id => byId.get(id));
+  const content = { summary: profile.summary, insights };
+  const references = new Set(insights.flatMap(claim => [...claim.supporting_refs, ...claim.contradicting_refs]));
+  return {
+    candidate_id: profileDigest({ input_digest: input.input_digest, ...content }),
+    ...content,
+    cited_evidence: [...references].map(ref => modelEvidence(input.evidence.get(ref))),
+  };
+}
+
+function checkText(value, field) {
+  if (typeof value !== "string" || !value.trim() || Array.from(value).length > 500 ||
+      /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f‪-‮⁦-⁩]/u.test(value)) {
+    throw new TypeError(`Each check issue needs a ${field} of at most 500 characters`);
+  }
+  return value.trim();
+}
+
+/**
+ * One judgment of the whole candidate. Only material problems are issues;
+ * citation completeness and phrasing are deliberately out of scope.
+ */
+export function validateListenerProfileCheck(candidate, value) {
+  if (value?.candidate_id !== candidate.candidate_id) {
+    throw new TypeError("The check must identify the current profile candidate");
+  }
+  if (!["pass", "revise"].includes(value.verdict)) throw new TypeError("The check verdict must be pass or revise");
+  const issues = value.issues ?? [];
+  if (!Array.isArray(issues) || issues.length > MAX_CHECK_ISSUES) {
+    throw new TypeError(`The check can list at most ${MAX_CHECK_ISSUES} material issues`);
+  }
+  if (value.verdict === "revise" && !issues.length) throw new TypeError("A revise verdict needs at least one material issue");
+  if (value.verdict === "pass" && issues.length) throw new TypeError("A pass verdict lists no issues; return revise for a material problem");
+  const targets = new Set(["summary", ...candidate.insights.map(claim => claim.claim_id)]);
+  return {
+    version: LISTENER_PROFILE_CHECK_VERSION,
+    candidate_id: candidate.candidate_id,
+    verdict: value.verdict,
+    issues: issues.map(issue => {
+      if (!targets.has(issue?.target)) {
+        throw new TypeError(`Check issue target ${JSON.stringify(issue?.target)} must be "summary" or a claim_id of this candidate`);
+      }
+      return { target: issue.target, problem: checkText(issue.problem, "problem"), correction: checkText(issue.correction, "correction") };
+    }),
+    checked_at: new Date().toISOString(),
+  };
+}
+
+export function searchListenerProfileEvidence(input, { section, query, offset = 0 } = {}) {
+  if (section !== undefined && !input.manifest.sections.some(item => item.section === section)) {
+    throw new TypeError("Unknown evidence section");
+  }
+  if ((query !== undefined && (typeof query !== "string" || query.length > 200)) ||
+      !Number.isSafeInteger(offset) || offset < 0) throw new TypeError("Invalid evidence search");
+  const needle = query?.normalize("NFKC").toLocaleLowerCase();
+  const matches = [...input.evidence.values()].filter(entry => !section || entry.section === section)
+    .filter(entry => !needle || JSON.stringify(modelEvidence(entry, 0, Number.MAX_SAFE_INTEGER).data)
+      .normalize("NFKC").toLocaleLowerCase().includes(needle));
+  return { total: matches.length, offset, items: matches.slice(offset, offset + 12).map(entry => modelEvidence(entry)),
+    next_offset: offset + 12 < matches.length ? offset + 12 : null };
+}

package/src/profile/listener-profile-worker.mjs

@@ -0,0 +1,179 @@
+import {
+  CHECKED_PROFILE_STATES, compileListenerProfile, listenerProfileDossier, LISTENER_PROFILE_SYNTHESIS_VERSION, modelEvidence,
+  sameListenerChoices,
+} from "./listener-profile-build.mjs";
+import {
+  LISTENER_PROFILE_CHECK_VERSION, listenerProfileCandidate, searchListenerProfileEvidence, validateListenerProfileCheck,
+} from "./listener-profile-verification.mjs";
+import { createConfiguredProfileBuildRuntime } from "../runtime/pi/profile-build-runtime.mjs";
+
+// One draft, one check, at most one revision and its check. Spare sessions let
+// a model that hits its turn limit continue the same stage.
+const MAX_REVISIONS = 1;
+const MAX_SESSIONS = 8;
+
+const HOUR_MS = 3_600_000;
+// A reading that only new plays have overtaken is rebuilt after a week or 300 plays.
+// Changed choices or analysis rebuild it at once. Any build attempt, the listener's
+// or this one, holds off the next background attempt for half a day.
+export const PROFILE_REFRESH = { listeningAgeMs: 7 * 24 * HOUR_MS, newPlays: 300, attemptGapMs: 12 * HOUR_MS };
+
+const synthesisNumber = (version) => Number(/^listener-synthesis\/(\d+)$/u.exec(version ?? "")?.[1] ?? Number.NaN);
+
+/** Rebuild a saved reading that newer evidence has overtaken, without being asked. */
+export async function refreshListenerProfileIfDue({ application, now = Date.now(), ...options } = {}) {
+  // The first reading is the listener's to ask for.
+  if (!application.profileBuildReady()) return { state: "unavailable" };
+  const reading = await application.getListenerProfile({ limit: 1 });
+  if (reading.state !== "stale") return { state: reading.state };
+  // A reading a newer Moondog wrote is left to that version or to the listener's own build, so an
+  // older terminal and a newer app never take turns rebuilding over each other's reading.
+  if (synthesisNumber(reading.synthesis_version) > synthesisNumber(LISTENER_PROFILE_SYNTHESIS_VERSION)) {
+    return { state: "newer_version" };
+  }
+  const { input, store } = application.getProfileBuildContext();
+  const lastAttempt = Date.parse(store.lastAttemptAt() ?? "");
+  if (now - lastAttempt < PROFILE_REFRESH.attemptGapMs) return { state: "waiting" };
+  if (reading.newer_listening_only) {
+    const newPlays = (input.manifest.coverage?.effective_listening_events ?? 0) -
+      (reading.coverage?.coverage?.effective_listening_events ?? 0);
+    if (now - Date.parse(reading.built_at) < PROFILE_REFRESH.listeningAgeMs && newPlays < PROFILE_REFRESH.newPlays) {
+      return { state: "waiting" };
+    }
+  }
+  return runListenerProfileBuild({ ...options, application });
+}
+
+export async function runListenerProfileBuild({
+  application, environment = process.env, signal, onProgress = () => {}, force = false,
+  runtimeFactory = createConfiguredProfileBuildRuntime,
+} = {}) {
+  signal?.throwIfAborted();
+  const { input, store } = application.getProfileBuildContext();
+  if (!input.evidence.size) return { state: "no_evidence" };
+  const previous = store.current();
+  if (!force && previous?.input_digest === input.input_digest && previous.synthesis_version === LISTENER_PROFILE_SYNTHESIS_VERSION &&
+      CHECKED_PROFILE_STATES.has(previous.verification?.state)) {
+    return { state: "unchanged", revision: await application.getListenerProfile() };
+  }
+  const runtime = await runtimeFactory(environment);
+  const status = runtime.publicStatus();
+  if (status.state !== "configured") throw new Error("Connect a model with /model, then use /profile build. Your imported profile is still available.");
+  const model = { provider: status.provider, model: status.model, worker_version: status.worker_version };
+  const build = store.begin(input, model, { force });
+  // Drafts and checks are checkpoints, never current profile revisions.
+  let finalization = build.finalization?.synthesis_version === LISTENER_PROFILE_SYNTHESIS_VERSION
+    ? structuredClone(build.finalization)
+    : { synthesis_version: LISTENER_PROFILE_SYNTHESIS_VERSION, draft: null, checks: [], revisions: 0 };
+  let compiled = finalization.draft ? compileListenerProfile(input, finalization.draft) : null;
+  let candidate = compiled ? listenerProfileCandidate(input, compiled) : null;
+  const latestCheck = () => finalization.checks.at(-1) ?? null;
+  const phase = () => !candidate ? "synthesis"
+    : latestCheck()?.candidate_id !== candidate.candidate_id ? "check"
+    : latestCheck().verdict === "revise" && finalization.revisions < MAX_REVISIONS ? "revision" : "complete";
+  const progress = () => ({ phase: phase(), checks: finalization.checks.length, revisions: finalization.revisions,
+    evidence_rows: input.evidence.size, resumed: build.resumed });
+  const inspected = new Set();
+  let stageFinished = false;
+  // Plays synced during a long build do not stop it: the reading is saved as current up to the
+  // plays it read. A changed Like or Avoid does, since the reading could contradict it.
+  const assertCurrentInput = () => {
+    signal?.throwIfAborted();
+    if (!sameListenerChoices(input.evidence, application.getProfileBuildContext().input.evidence)) {
+      throw new Error("Your choices changed during the build. Use /profile build to read the updated evidence.");
+    }
+  };
+  // The checkpoint commits before memory changes, so a lost lease leaves both unchanged.
+  const save = (next, apply = () => {}) => {
+    store.checkpointFinalization(build, next);
+    apply();
+    finalization = next;
+    stageFinished = true;
+    onProgress(progress());
+  };
+  const session = {
+    phase: phase(), manifest: input.manifest, dossier: listenerProfileDossier(input), progress,
+    language: application.locale ?? "en",
+    get candidate() { return candidate; },
+    get check() { return latestCheck(); },
+    search: args => {
+      signal?.throwIfAborted();
+      const result = searchListenerProfileEvidence(input, args);
+      for (const item of result.items) inspected.add(item.reference_id);
+      return result;
+    },
+    evidence: async (referenceId, offset = 0) => {
+      signal?.throwIfAborted();
+      if (!Number.isSafeInteger(offset) || offset < 0) throw new TypeError("Invalid evidence offset");
+      const entry = input.evidence.get(referenceId);
+      if (!entry) throw new TypeError("Unknown profile evidence reference");
+      const result = modelEvidence(entry, offset);
+      if (entry.data.evidence_id) {
+        try {
+          const detail = await application.explainProfileEvidence({ evidenceId: entry.data.evidence_id });
+          // The prose basis restates the row's measures, including any this input leaves out as unrecorded.
+          result.explanation = { ...(input.unrecorded?.has(referenceId) ? {} : { basis: detail.basis_summary }),
+            limitation: detail.interpretation_limit ?? detail.limitation };
+        } catch { /* The frozen row remains the source for aggregate-only evidence. */ }
+      }
+      inspected.add(referenceId);
+      return result;
+    },
+    modelRetry: retry => { if (!signal?.aborted) onProgress({ ...progress(), model_retry: retry }); },
+    submit: value => {
+      signal?.throwIfAborted();
+      if (!["synthesis", "revision"].includes(session.phase)) throw new Error("Submit a profile only while writing or revising it");
+      if (stageFinished) throw new Error("Submit one complete profile per session");
+      const next = compileListenerProfile(input, value);
+      const nextCandidate = listenerProfileCandidate(input, next);
+      save({ ...finalization, draft: { summary: nextCandidate.summary, insights: nextCandidate.insights },
+        revisions: finalization.revisions + Number(session.phase === "revision") }, () => {
+        compiled = next;
+        candidate = nextCandidate;
+      });
+      return { state: "draft_saved", next: "check" };
+    },
+    record: value => {
+      signal?.throwIfAborted();
+      if (session.phase !== "check" || stageFinished) throw new Error("Record one check in the separate check session");
+      const check = validateListenerProfileCheck(candidate, value);
+      save({ ...finalization, checks: [...finalization.checks, check] });
+      return { verdict: check.verdict, issues: check.issues.length };
+    },
+  };
+  const abort = () => runtime.abort?.();
+  signal?.addEventListener("abort", abort, { once: true });
+  try {
+    signal?.throwIfAborted();
+    onProgress(progress());
+    let sessions = 0;
+    while (phase() !== "complete") {
+      signal?.throwIfAborted();
+      session.phase = phase();
+      stageFinished = false;
+      if (++sessions > MAX_SESSIONS) {
+        throw new Error("The profile build reached its session limit. Your previous reading and the saved draft are kept. Use /profile build to continue.");
+      }
+      const inspectedBefore = inspected.size;
+      await runtime.investigate(session);
+      signal?.throwIfAborted();
+      assertCurrentInput();
+      if (!stageFinished && inspected.size === inspectedBefore) {
+        throw new Error("The profile model stopped making progress. Your previous reading and any saved draft are kept. Use /profile build to retry.");
+      }
+    }
+    const check = latestCheck();
+    store.complete(build, { ...compiled, verification: {
+      version: LISTENER_PROFILE_CHECK_VERSION,
+      state: check.verdict === "pass" ? (finalization.revisions ? "revised" : "passed") : "flagged",
+      revisions: finalization.revisions, checks: finalization.checks,
+    } }, assertCurrentInput);
+    return { state: "ready", revision: await application.getListenerProfile(), ...progress() };
+  } catch (error) {
+    store.interrupt(build);
+    throw error;
+  } finally {
+    signal?.removeEventListener("abort", abort);
+    runtime.abort?.();
+  }
+}