daftari 1.29.0 → 1.31.0

package/dist/tools/themes.js

@@ -1,33 +1,42 @@
-// vault_themes — thematic clustering over document-pooled embeddings.
+// vault_themes — thematic clustering over chunk embeddings with per-document
+// theme distributions (#58).
 //
-// Cluster DOCUMENTS, not chunks: for each document we mean-pool its chunk
-// embeddings into one vector, L2-normalise it, and cluster the resulting
-// ~N-doc set. ~3.5k document vectors instead of ~44k chunk vectors makes
-// every algorithm cheap (silhouette becomes tractable on the full set; no
-// sampling needed at this scale).
+// Cluster CHUNKS, not pooled documents: v1 mean-pooled each document's chunk
+// embeddings into one vector, which collapses a genuinely two-region document
+// (half pricing, half moonshot) into a single point that lands in neither
+// region. v2 clusters the chunk vectors directly and aggregates chunk
+// assignments into per-document DISTRIBUTIONS over themes, so that document
+// carries weight in both themes instead of a wrong single assignment.
 //
-// v1 constraints (locked):
-//   - One-doc-one-theme. Each document lives in exactly one cluster.
+// v2 constraints (locked):
+//   - Documents have distributions, not single assignments. `documentCount`
+//     is COVERAGE (a doc in two themes counts in both); the partition lives
+//     in `primaryDocumentCount` (each doc counted once, at its argmax theme).
+//   - A theme membership below MEMBERSHIP_MIN_FRACTION of the doc's chunks
+//     is dropped (except the argmax) — a one-chunk aside must not create a
+//     phantom membership (#58's granularity concern).
 //   - Heuristic labels from TF-IDF over titles + tags. No LLM call.
 //   - No new storage — the tool is read-only against the existing
 //     embeddings / chunks / documents tables.
-//   - Default k-sweep over {10, 15, 20, 25}; an explicit `k` argument
-//     skips the sweep.
-//   - Deterministic via a fixed seed for the k-means RNG.
+//   - Default k-sweep over {10, 15, 20, 25}; an explicit `k` argument skips
+//     the sweep. Silhouette for the sweep is computed on a deterministic
+//     stride sample (chunk-scale inputs make the full O(n²) score ~2B pairs).
+//   - Deterministic via a fixed seed for the k-means RNG and stride (never
+//     random) sampling.
 import { canRead } from "../access/rbac.js";
 import { err, ok } from "../frontmatter/types.js";
 import { getProvider } from "../search/vector.js";
 import { blobToEmbedding } from "../storage/index-db.js";
-import { clusterCoherence, kmeans, meanPoolL2, pickK, seededRng, selectSecondaryMemberships, } from "../themes/clustering.js";
+import { clusterCoherence, kmeans, membershipDistributions, pickK, seededRng, strideSample, } from "../themes/clustering.js";
 import { ensureIndexReady, openIndexForActiveProvider } from "./search.js";
 // Fixed seed for the k-means RNG. The whole point of vault_themes is that
 // the same vault produces the same themes — a random seed would give the
 // user different themes every run and erode trust in the output.
 const THEMES_RNG_SEED = 0x7da17f1; // arbitrary; constant is what matters
 // Default k-sweep range (from the issue dialogue). Each is clamped to the
-// number of clusterable documents inside pickK.
+// number of clusterable chunks inside pickK.
 const DEFAULT_K_CANDIDATES = [10, 15, 20, 25];
-// k-means iteration cap. Documents on the unit sphere converge fast in
+// k-means iteration cap. Vectors on the unit sphere converge fast in
 // practice; this is a safety bound for pathological inputs.
 const KMEANS_MAX_ITER = 50;
 // Representative-doc count returned per theme. 5 is enough to be useful in
@@ -36,15 +45,19 @@ const REPRESENTATIVE_DOCS_PER_THEME = 5;
 // Tag-frequency cutoff per theme. Five most-common tags is enough signal
 // without burying the user in noise.
 const RELATED_TAGS_PER_THEME = 5;
-// Secondary-membership tuning. A doc qualifies as a secondary member of a
-// non-primary cluster when its centroid similarity is BOTH within DELTA of
-// the primary AND above MIN_ABS_SIMILARITY. The cap prevents a centroid-
-// of-mass doc from showing up in every cluster. Defaults are deliberately
-// conservative — secondaries should surface cross-cutting docs, not bury
-// the primary signal.
-const SECONDARY_DELTA = 0.1;
-const SECONDARY_MIN_SIMILARITY = 0.5;
-const SECONDARY_MAX_PER_DOC = 2;
+// A doc is a member of a theme when at least this fraction of its chunks
+// landed there (the argmax theme always qualifies). 0.25 keeps the 60/40
+// two-region synthesis doc in both themes while dropping a one-chunk aside
+// in a ten-chunk doc (0.1) — #58's phantom-theme guard.
+const MEMBERSHIP_MIN_FRACTION = 0.25;
+// Silhouette sample cap for the k-sweep. The sweep clusters every chunk but
+// scores k on a deterministic stride sample: full silhouette is O(n²) and
+// chunk-scale inputs (~44k on the motivating vault) would make it ~2B pairs
+// per candidate k.
+const SILHOUETTE_SAMPLE_CAP = 2000;
+// Per-theme coherence sample cap, same O(n²) rationale as the silhouette
+// cap but per cluster.
+const COHERENCE_SAMPLE_CAP = 200;
 // Hard cap on secondaries reported per theme. Same UX rationale as
 // REPRESENTATIVE_DOCS_PER_THEME — enough to be useful, bounded enough to
 // not bury the user.
@@ -95,14 +108,28 @@ function labelTokens(text) {
 }
 // Loads every (path, content_hash, embedding) row for the active model, in
 // path-then-chunk-index order, and groups embeddings by document path.
-function loadEmbeddingsByPath(db, model) {
-    const rows = db
-        .prepare(`SELECT c.path AS path, e.embedding AS embedding, e.dim AS dim
+//
+// When `collection` is given the chunk scan is joined to `documents` and
+// filtered to that collection IN SQL, so out-of-scope embeddings are never
+// read out of the database or copied into memory (E3). The collection column
+// is authoritative for scope — the same predicate `buildScopedDocs` applies —
+// so pushing it down here changes nothing about the clustered result, only
+// which rows are loaded.
+function loadEmbeddingsByPath(db, model, collection) {
+    const sql = collection
+        ? `SELECT c.path AS path, e.embedding AS embedding, e.dim AS dim
+         FROM chunks c
+         JOIN documents d ON d.path = c.path
+         LEFT JOIN embeddings e
+           ON e.content_hash = c.content_hash AND e.model = ?
+        WHERE d.collection = ?
+        ORDER BY c.path, c.chunk_index`
+        : `SELECT c.path AS path, e.embedding AS embedding, e.dim AS dim
          FROM chunks c
          LEFT JOIN embeddings e
            ON e.content_hash = c.content_hash AND e.model = ?
-        ORDER BY c.path, c.chunk_index`)
-        .all(model);
+        ORDER BY c.path, c.chunk_index`;
+    const rows = (collection ? db.prepare(sql).all(model, collection) : db.prepare(sql).all(model));
     const provider = getProvider();
     const expectedDim = provider.dim;
     const out = new Map();
@@ -124,49 +151,150 @@ function loadEmbeddingsByPath(db, model) {
     }
     return out;
 }
-// Applies the scope filter (collection, tags), RBAC, and the doc-has-an-
-// embedded-chunk requirement; returns the clusterable doc set and the count
-// of docs the embedding requirement skipped.
-function buildScopedDocs(documents, embeddingsByPath, filters, access) {
-    const scoped = [];
-    let skipped = 0;
-    for (const doc of documents) {
-        if (filters.collection && doc.collection !== filters.collection)
+// Process-level memo of the chunk vector set. Keyed by
+// `${vaultRoot} ${model} ${collection ?? "*"}`; the entry's
+// `signature` is checked against the live index signature on every call and a
+// mismatch (any reindex/content change) discards the stale entry. Bounded to
+// one entry per scope key — the working set for a single vault is the same
+// vectors clustering must hold in memory anyway, and the cache is dropped
+// wholesale on signature change.
+const chunkSetCache = new Map();
+// Test-only hook: reset the process-level cache between cases.
+export function __resetThemesCache() {
+    chunkSetCache.clear();
+}
+// Cheap content signature for the (collection-scoped) chunk+embedding set of
+// the active model. It changes whenever a chunk is added/removed/re-hashed or
+// an embedding is (re)written, which is exactly when the chunk vectors would
+// differ — so a matching signature guarantees the cached set is still valid.
+// `documents.rowid` participation via count keeps a bare collection rename
+// honest. This scans indexes, not the embedding blobs.
+function indexSignature(db, model, collection) {
+    const sql = collection
+        ? `SELECT COUNT(*) AS n,
+              group_concat(c.content_hash, '') AS hashes,
+              SUM(CASE WHEN e.content_hash IS NULL THEN 0 ELSE 1 END) AS embn
+         FROM chunks c
+         JOIN documents d ON d.path = c.path
+         LEFT JOIN embeddings e
+           ON e.content_hash = c.content_hash AND e.model = ?
+        WHERE d.collection = ?`
+        : `SELECT COUNT(*) AS n,
+              group_concat(c.content_hash, '') AS hashes,
+              SUM(CASE WHEN e.content_hash IS NULL THEN 0 ELSE 1 END) AS embn
+         FROM chunks c
+         LEFT JOIN embeddings e
+           ON e.content_hash = c.content_hash AND e.model = ?`;
+    const row = (collection ? db.prepare(sql).get(model, collection) : db.prepare(sql).get(model));
+    // A tiny FNV-1a over the concatenated hashes keeps the key bounded regardless
+    // of vault size while still varying on any content change.
+    const hashes = row.hashes ?? "";
+    let h = 0x811c9dc5;
+    for (let i = 0; i < hashes.length; i++) {
+        h ^= hashes.charCodeAt(i);
+        h = Math.imul(h, 0x01000193);
+    }
+    const hashDigest = (h >>> 0).toString(16);
+    return `${model}:${collection ?? "*"}:${row.n ?? 0}:${row.embn ?? 0}:${hashDigest}`;
+}
+// Returns the per-doc chunk vector set for the given scope, using the cache
+// when the live signature matches and rebuilding (load + normalise)
+// otherwise. This is the load-avoidance core of the E3 fix: a second call
+// against an unchanged index re-uses the vectors instead of re-reading every
+// embedding blob.
+function getChunkSet(db, vaultRoot, model, documentsByPath, collection) {
+    const cacheKey = `${vaultRoot} ${model} ${collection ?? "*"}`;
+    const signature = indexSignature(db, model, collection);
+    const cached = chunkSetCache.get(cacheKey);
+    if (cached && cached.signature === signature)
+        return cached;
+    const embeddingsByPath = loadEmbeddingsByPath(db, model, collection);
+    const docs = [];
+    const unembeddedPaths = new Set();
+    for (const [path, doc] of documentsByPath) {
+        if (collection && doc.collection !== collection)
             continue;
-        if (filters.tags && filters.tags.length > 0) {
-            const hasAll = filters.tags.every((t) => doc.tags.includes(t));
-            if (!hasAll)
+        const raw = embeddingsByPath.get(path) ?? [];
+        // Normalise each chunk onto the unit sphere (cosine semantics for the
+        // clustering); drop zero vectors — they carry no direction to cluster.
+        const chunks = [];
+        for (const vec of raw) {
+            // Single pass per chunk: this runs at chunk scale (~44k on the
+            // motivating vault), so the norm is computed once and reused for the
+            // scaling rather than recomputed inside a normalize helper.
+            let norm = 0;
+            for (let i = 0; i < vec.length; i++) {
+                const x = vec[i];
+                norm += x * x;
+            }
+            if (norm === 0)
                 continue;
+            const inv = 1 / Math.sqrt(norm);
+            const out = new Float32Array(vec.length);
+            for (let i = 0; i < vec.length; i++)
+                out[i] = vec[i] * inv;
+            chunks.push(out);
         }
-        if (access && !canRead(access.role, doc.collection))
-            continue;
-        const chunkVecs = embeddingsByPath.get(doc.path) ?? [];
-        const pooled = meanPoolL2(chunkVecs);
-        if (!pooled) {
-            // No embedded chunks (or an all-zero pool): count as skipped per
-            // the v1 contract.
-            skipped += 1;
+        if (chunks.length === 0) {
+            unembeddedPaths.add(path);
             continue;
         }
+        docs.push({
+            path: doc.path,
+            title: doc.title,
+            collection: doc.collection,
+            tags: doc.tags,
+            chunks,
+        });
+    }
+    const set = { signature, docs, unembeddedPaths };
+    chunkSetCache.set(cacheKey, set);
+    return set;
+}
+// Applies the per-caller scope (tags, RBAC) and the doc-has-an-embedded-chunk
+// requirement over a pre-loaded set. The collection filter is already applied
+// upstream in `getChunkSet` (and pushed into SQL), so it is not re-checked
+// here beyond honouring it for the skipped count. Returns the clusterable doc
+// set and the count of docs the embedding requirement skipped.
+//
+// `skipped` must match the pre-cache contract exactly: a doc counts as skipped
+// only if it passes collection + tags + RBAC and yet has no chunk vector.
+function finalizeScope(chunkSet, documentsByPath, filters, access) {
+    const passesTagsAndRbac = (collection, tags) => {
+        if (filters.tags && filters.tags.length > 0) {
+            const hasAll = filters.tags.every((t) => tags.includes(t));
+            if (!hasAll)
+                return false;
+        }
+        if (access && !canRead(access.role, collection))
+            return false;
+        return true;
+    };
+    const scoped = [];
+    for (const doc of chunkSet.docs) {
+        if (!passesTagsAndRbac(doc.collection, doc.tags))
+            continue;
         scoped.push({
             path: doc.path,
             title: doc.title,
             collection: doc.collection,
             tags: doc.tags,
-            vector: pooled,
+            chunks: doc.chunks,
         });
     }
+    let skipped = 0;
+    for (const path of chunkSet.unembeddedPaths) {
+        const doc = documentsByPath.get(path);
+        if (!doc)
+            continue;
+        if (filters.collection && doc.collection !== filters.collection)
+            continue;
+        if (!passesTagsAndRbac(doc.collection, doc.tags))
+            continue;
+        skipped += 1;
+    }
     return { scoped, skipped };
 }
-// Cosine similarity for L2-normalised vectors == dot product. Used for
-// representative-doc ranking against a cluster centroid.
-function dotNormalized(a, b) {
-    let s = 0;
-    const n = Math.min(a.length, b.length);
-    for (let i = 0; i < n; i++)
-        s += a[i] * b[i];
-    return s;
-}
 // TF-IDF over the cluster: term frequency among the cluster's titles +
 // tags, divided by the document-frequency across ALL clusters (so a term
 // that names every theme — e.g. "doc" — doesn't dominate any one of them).
@@ -194,7 +322,7 @@ function buildLabel(clusterDocs, globalDocCount, globalDf, themeIndex) {
             continue;
         scored.push({ term, score: freq * idf });
     }
-    scored.sort((a, b) => b.score - a.score);
+    scored.sort((a, b) => b.score - a.score || a.term.localeCompare(b.term));
     const top = scored.slice(0, 3).map((s) => s.term);
     if (top.length > 0)
         return top.join(" / ");
@@ -241,13 +369,6 @@ function topTagsForCluster(clusterDocs) {
         .slice(0, RELATED_TAGS_PER_THEME)
         .map(([t]) => t);
 }
-function representativesForCluster(clusterDocs, centroid) {
-    return [...clusterDocs]
-        .map((d) => ({ path: d.path, sim: dotNormalized(d.vector, centroid) }))
-        .sort((a, b) => b.sim - a.sim || a.path.localeCompare(b.path))
-        .slice(0, REPRESENTATIVE_DOCS_PER_THEME)
-        .map((d) => d.path);
-}
 function parseArgs(args) {
     let k;
     if (args.k !== undefined) {
@@ -278,104 +399,180 @@ export async function vaultThemes(vaultRoot, args, access) {
     const db = dbResult.value;
     try {
         const provider = getProvider();
-        const docRows = db
-            .prepare("SELECT path, title, collection, tags FROM documents ORDER BY path")
-            .all();
-        const documents = docRows.map((r) => ({
-            path: r.path,
-            title: r.title,
-            collection: r.collection,
-            domain: "",
-            status: "",
-            confidence: "",
-            updated: "",
-            tags: JSON.parse(r.tags),
-            content: "",
-            tokens: [],
-            ttlDays: null,
-            created: "",
-            supersededBy: null,
-        }));
-        const embeddingsByPath = loadEmbeddingsByPath(db, provider.id);
-        const { scoped, skipped } = buildScopedDocs(documents, embeddingsByPath, parsed.value, access);
+        // Push the collection filter into the documents scan too: a scoped call
+        // never materialises out-of-scope document rows, and `JSON.parse(tags)`
+        // runs only for in-scope docs (part of the E3 load-avoidance).
+        const collection = parsed.value.collection;
+        const docRows = (collection
+            ? db
+                .prepare("SELECT path, title, collection, tags FROM documents WHERE collection = ? ORDER BY path")
+                .all(collection)
+            : db.prepare("SELECT path, title, collection, tags FROM documents ORDER BY path").all());
+        const documentsByPath = new Map();
+        for (const r of docRows) {
+            documentsByPath.set(r.path, {
+                path: r.path,
+                title: r.title,
+                collection: r.collection,
+                domain: "",
+                status: "",
+                confidence: "",
+                updated: "",
+                tags: JSON.parse(r.tags),
+                content: "",
+                tokens: [],
+                ttlDays: null,
+                created: "",
+                supersededBy: null,
+            });
+        }
+        const chunkSet = getChunkSet(db, vaultRoot, provider.id, documentsByPath, collection);
+        const { scoped, skipped } = finalizeScope(chunkSet, documentsByPath, parsed.value, access);
         if (scoped.length === 0) {
             return ok({
                 themes: [],
+                docMemberships: {},
                 totalDocuments: 0,
+                totalChunks: 0,
                 skippedDocuments: skipped,
                 selectedK: 0,
+                droppedClusters: 0,
                 clusteredAt: new Date().toISOString(),
             });
         }
-        const vectors = scoped.map((s) => s.vector);
+        // Flatten to chunk-level arrays: the clustering input plus the chunk→doc
+        // map the distribution aggregation needs.
+        const chunkVectors = [];
+        const chunkDoc = [];
+        for (let d = 0; d < scoped.length; d++) {
+            for (const vec of scoped[d].chunks) {
+                chunkVectors.push(vec);
+                chunkDoc.push(d);
+            }
+        }
         const rng = seededRng(THEMES_RNG_SEED);
         let selectedK;
         let assignments;
-        let centroids;
         if (parsed.value.k !== undefined) {
-            const result = kmeans(vectors, parsed.value.k, rng, KMEANS_MAX_ITER);
+            const result = kmeans(chunkVectors, parsed.value.k, rng, KMEANS_MAX_ITER);
             selectedK = result.centroids.length;
             assignments = result.assignments;
-            centroids = result.centroids;
         }
         else {
-            const result = pickK(vectors, DEFAULT_K_CANDIDATES, rng, KMEANS_MAX_ITER);
+            const result = pickK(chunkVectors, DEFAULT_K_CANDIDATES, rng, KMEANS_MAX_ITER, SILHOUETTE_SAMPLE_CAP);
             selectedK = result.centroids.length;
             assignments = result.assignments;
-            centroids = result.centroids;
         }
-        // Group docs by cluster.
-        const byCluster = new Map();
-        for (let i = 0; i < scoped.length; i++) {
+        // Aggregate chunk assignments into per-doc theme distributions (#58).
+        const distributions = membershipDistributions(assignments, chunkDoc, scoped.length, MEMBERSHIP_MIN_FRACTION);
+        // Group member/primary docs by cluster from the distributions.
+        const membersByCluster = new Map();
+        const primaryByCluster = new Map();
+        for (let d = 0; d < distributions.length; d++) {
+            const memberships = distributions[d] ?? [];
+            for (let m = 0; m < memberships.length; m++) {
+                const { cluster, weight } = memberships[m];
+                const list = membersByCluster.get(cluster);
+                if (list)
+                    list.push({ docIndex: d, weight });
+                else
+                    membersByCluster.set(cluster, [{ docIndex: d, weight }]);
+                // memberships are weight-desc with deterministic ties: entry 0 IS
+                // the argmax/primary.
+                if (m === 0) {
+                    const plist = primaryByCluster.get(cluster);
+                    if (plist)
+                        plist.push(d);
+                    else
+                        primaryByCluster.set(cluster, [d]);
+                }
+            }
+        }
+        // Chunk vectors per cluster for the coherence score.
+        const chunksByCluster = new Map();
+        for (let i = 0; i < chunkVectors.length; i++) {
             const c = assignments[i];
-            const doc = scoped[i];
-            const list = byCluster.get(c);
+            const list = chunksByCluster.get(c);
             if (list)
-                list.push(doc);
+                list.push(chunkVectors[i]);
             else
-                byCluster.set(c, [doc]);
+                chunksByCluster.set(c, [chunkVectors[i]]);
+        }
+        // A chunk-bearing cluster no doc's retained membership points at is a
+        // grab-bag of sub-threshold asides — dropped from `themes` by the
+        // phantom-theme guard, but COUNTED so the omission is visible (empty
+        // k-means clusters from the k-clamp are not "dropped": they never had
+        // chunks).
+        let droppedClusters = 0;
+        for (const clusterId of chunksByCluster.keys()) {
+            if (!membersByCluster.has(clusterId))
+                droppedClusters += 1;
         }
         const globalDf = buildGlobalDf(scoped);
-        // Secondary memberships: for each doc, find non-primary clusters its
-        // pooled vector also aligns with. This is the soft-reporting layer on
-        // top of the hard partition — it does NOT change documentCount, which
-        // still reflects primary membership only.
-        const secondaryByCluster = selectSecondaryMemberships(vectors, assignments, centroids, {
-            delta: SECONDARY_DELTA,
-            minSimilarity: SECONDARY_MIN_SIMILARITY,
-            maxPerDoc: SECONDARY_MAX_PER_DOC,
-        });
         const themes = [];
         let themeIndex = 0;
-        for (const [clusterIdx, clusterDocs] of byCluster) {
-            if (clusterDocs.length === 0)
+        // Iterate clusters in index order so labels/fallbacks are deterministic.
+        const clusterIds = [...membersByCluster.keys()].sort((a, b) => a - b);
+        for (const clusterId of clusterIds) {
+            const members = membersByCluster.get(clusterId) ?? [];
+            if (members.length === 0)
                 continue;
-            const centroid = centroids[clusterIdx];
-            if (!centroid)
-                continue;
-            // Singleton clusters have no pairs to average — return null rather
-            // than the mathematically-trivial 1.0, which would falsely imply
-            // tightness.
-            const coherence = clusterDocs.length < 2 ? null : clusterCoherence(clusterDocs.map((d) => d.vector));
-            const secondaries = (secondaryByCluster.get(clusterIdx) ?? [])
-                .slice(0, SECONDARY_DOCS_PER_THEME)
-                .map((s) => scoped[s.docIndex].path);
+            members.sort((a, b) => b.weight - a.weight ||
+                scoped[a.docIndex].path.localeCompare(scoped[b.docIndex].path));
+            const primaries = new Set(primaryByCluster.get(clusterId) ?? []);
+            // Label/tag signal comes from the docs that primarily live here when
+            // any do — a theme should be named by its residents, not its visitors.
+            const primaryDocs = [...primaries].map((d) => scoped[d]);
+            const labelDocs = primaryDocs.length > 0 ? primaryDocs : members.map((m) => scoped[m.docIndex]);
+            const clusterChunks = chunksByCluster.get(clusterId) ?? [];
+            const coherenceChunks = clusterChunks.length > COHERENCE_SAMPLE_CAP
+                ? strideSample(clusterChunks.length, COHERENCE_SAMPLE_CAP).map((i) => clusterChunks[i])
+                : clusterChunks;
+            // A single-chunk theme has no pairs to average — null rather than the
+            // mathematically-trivial 1.0, which would falsely imply tightness.
+            const coherence = coherenceChunks.length < 2 ? null : clusterCoherence(coherenceChunks);
             themes.push({
-                label: buildLabel(clusterDocs, scoped.length, globalDf, themeIndex),
-                documentCount: clusterDocs.length,
+                id: clusterId,
+                label: buildLabel(labelDocs, scoped.length, globalDf, themeIndex),
+                documentCount: members.length,
+                primaryDocumentCount: primaries.size,
                 coherence,
-                representativeDocs: representativesForCluster(clusterDocs, centroid),
-                secondaryDocs: secondaries,
-                relatedTags: topTagsForCluster(clusterDocs),
+                // Residents only: a high-weight visitor ranks in `members` but must
+                // not appear as a representative AND a secondary of the same theme —
+                // the disjointness invariant v1 held and tests pin.
+                representativeDocs: members
+                    .filter((m) => primaries.has(m.docIndex))
+                    .slice(0, REPRESENTATIVE_DOCS_PER_THEME)
+                    .map((m) => scoped[m.docIndex].path),
+                secondaryDocs: members
+                    .filter((m) => !primaries.has(m.docIndex))
+                    .slice(0, SECONDARY_DOCS_PER_THEME)
+                    .map((m) => scoped[m.docIndex].path),
+                relatedTags: topTagsForCluster(labelDocs),
             });
             themeIndex += 1;
         }
         themes.sort((a, b) => b.documentCount - a.documentCount || a.label.localeCompare(b.label));
+        // Cross-cutting docs only: a single-theme doc's distribution is trivial,
+        // and listing every path would bloat the payload at vault scale.
+        const docMemberships = {};
+        for (let d = 0; d < distributions.length; d++) {
+            const memberships = distributions[d] ?? [];
+            if (memberships.length < 2)
+                continue;
+            docMemberships[scoped[d].path] = memberships.map((m) => ({
+                theme: m.cluster,
+                weight: m.weight,
+            }));
+        }
         return ok({
             themes,
+            docMemberships,
             totalDocuments: scoped.length,
+            totalChunks: chunkVectors.length,
             skippedDocuments: skipped,
             selectedK,
+            droppedClusters,
             clusteredAt: new Date().toISOString(),
         });
     }
@@ -391,15 +588,18 @@ export const themesTools = [
         name: "vault_themes",
         title: "Cluster vault themes",
         annotations: { readOnlyHint: true },
-        description: "Surface thematic clusters across the vault using k-means over " +
-            "document-pooled embeddings. Each document's chunk vectors are " +
-            "mean-pooled into one vector, L2-normalised, and clustered. By " +
-            "default the tool sweeps k ∈ {10, 15, 20, 25} and picks the k with " +
-            "the best mean silhouette; pass `k` to skip the sweep. Each theme " +
-            "reports a heuristic label (TF-IDF over titles + tags), a coherence " +
-            "score (mean pairwise cosine inside the cluster), representative " +
-            "documents nearest the centroid, and the most frequent tags. " +
-            "Output is deterministic for the same vault.",
+        description: "Surface thematic clusters across the vault using k-means over CHUNK " +
+            "embeddings, with per-document theme distributions. A document whose " +
+            "chunks split across two clusters is a member of both themes " +
+            "(docMemberships reports its weights); documentCount is coverage (a " +
+            "two-theme doc counts in both) while primaryDocumentCount partitions " +
+            "docs by their dominant theme. By default the tool sweeps " +
+            "k ∈ {10, 15, 20, 25} and picks the k with the best (sampled) mean " +
+            "silhouette; pass `k` to skip the sweep. Each theme reports a " +
+            "heuristic label (TF-IDF over titles + tags), a coherence score (mean " +
+            "pairwise cosine among its chunks), representative documents ranked " +
+            "by membership weight, and the most frequent tags. Output is " +
+            "deterministic for the same vault.",
         inputSchema: {
             type: "object",
             properties: {