@ghcrawl/api-core 0.2.0 → 0.4.0

package/dist/service.js

@@ -1,7 +1,7 @@
 import http from 'node:http';
 import crypto from 'node:crypto';
 import { IterableMapper } from '@shutterstock/p-map-iterable';
-import { actionResponseSchema, clusterDetailResponseSchema, clusterResultSchema, clusterSummariesResponseSchema, clustersResponseSchema, embedResultSchema, healthResponseSchema, neighborsResponseSchema, refreshResponseSchema, repositoriesResponseSchema, searchResponseSchema, syncResultSchema, threadsResponseSchema, } from '@ghcrawl/api-contract';
+import { actionResponseSchema, authorThreadsResponseSchema, closeResponseSchema, clusterDetailResponseSchema, clusterResultSchema, clusterSummariesResponseSchema, clustersResponseSchema, embedResultSchema, healthResponseSchema, neighborsResponseSchema, refreshResponseSchema, repositoriesResponseSchema, searchResponseSchema, syncResultSchema, threadsResponseSchema, } from '@ghcrawl/api-contract';
 import { buildClusters } from './cluster/build.js';
 import { ensureRuntimeDirs, isLikelyGitHubToken, isLikelyOpenAiApiKey, loadConfig, requireGithubToken, requireOpenAiKey, } from './config.js';
 import { migrate } from './db/migrate.js';
@@ -13,6 +13,7 @@ import { cosineSimilarity, rankNearestNeighbors } from './search/exact.js';
 const SYNC_BATCH_SIZE = 100;
 const SYNC_BATCH_DELAY_MS = 5000;
 const STALE_CLOSED_SWEEP_LIMIT = 1000;
+const CLUSTER_PROGRESS_INTERVAL_MS = 5000;
 const EMBED_ESTIMATED_CHARS_PER_TOKEN = 3;
 const EMBED_MAX_ITEM_TOKENS = 7000;
 const EMBED_MAX_BATCH_TOKENS = 250000;
@@ -26,6 +27,9 @@ function parseIso(value) {
     const parsed = Date.parse(value);
     return Number.isNaN(parsed) ? null : parsed;
 }
+function isEffectivelyClosed(row) {
+    return row.state !== 'open' || row.closed_at_local !== null;
+}
 function isMissingGitHubResourceError(error) {
     const status = typeof error?.status === 'number' ? Number(error.status) : null;
     if (status === 404 || status === 410) {
@@ -149,6 +153,10 @@ function threadToDto(row, clusterId) {
         number: row.number,
         kind: row.kind,
         state: row.state,
+        isClosed: isEffectivelyClosed(row),
+        closedAtGh: row.closed_at_gh ?? null,
+        closedAtLocal: row.closed_at_local ?? null,
+        closeReasonLocal: row.close_reason_local ?? null,
         title: row.title,
         body: row.body,
         authorLogin: row.author_login,
@@ -259,17 +267,177 @@ export class GHCrawlService {
             .all(repository.id, repository.id);
         for (const row of clusterRows)
             clusterIds.set(row.thread_id, row.cluster_id);
-        let sql = "select * from threads where repo_id = ? and state = 'open'";
+        let sql = 'select * from threads where repo_id = ?';
         const args = [repository.id];
+        if (!params.includeClosed) {
+            sql += " and state = 'open' and closed_at_local is null";
+        }
         if (params.kind) {
             sql += ' and kind = ?';
             args.push(params.kind);
         }
+        if (params.numbers && params.numbers.length > 0) {
+            const uniqueNumbers = Array.from(new Set(params.numbers.filter((value) => Number.isSafeInteger(value) && value > 0)));
+            if (uniqueNumbers.length === 0) {
+                return threadsResponseSchema.parse({
+                    repository,
+                    threads: [],
+                });
+            }
+            sql += ` and number in (${uniqueNumbers.map(() => '?').join(', ')})`;
+            args.push(...uniqueNumbers);
+        }
         sql += ' order by updated_at_gh desc, number desc';
         const rows = this.db.prepare(sql).all(...args);
+        const orderedRows = params.numbers && params.numbers.length > 0
+            ? (() => {
+                const byNumber = new Map(rows.map((row) => [row.number, row]));
+                const uniqueRequested = Array.from(new Set(params.numbers));
+                return uniqueRequested.map((number) => byNumber.get(number)).filter((row) => row !== undefined);
+            })()
+            : rows;
         return threadsResponseSchema.parse({
             repository,
-            threads: rows.map((row) => threadToDto(row, clusterIds.get(row.id) ?? null)),
+            threads: orderedRows.map((row) => threadToDto(row, clusterIds.get(row.id) ?? null)),
+        });
+    }
+    listAuthorThreads(params) {
+        const repository = this.requireRepository(params.owner, params.repo);
+        const normalizedLogin = params.login.trim();
+        if (!normalizedLogin) {
+            return authorThreadsResponseSchema.parse({
+                repository,
+                authorLogin: '',
+                threads: [],
+            });
+        }
+        const clusterIds = new Map();
+        const clusterRows = this.db
+            .prepare(`select cm.thread_id, cm.cluster_id
+         from cluster_members cm
+         join clusters c on c.id = cm.cluster_id
+         where c.repo_id = ? and c.cluster_run_id = (
+           select id from cluster_runs where repo_id = ? and status = 'completed' order by id desc limit 1
+         )`)
+            .all(repository.id, repository.id);
+        for (const row of clusterRows)
+            clusterIds.set(row.thread_id, row.cluster_id);
+        const rows = this.db
+            .prepare(`select *
+         from threads
+         where repo_id = ? and lower(author_login) = lower(?)
+           ${params.includeClosed ? '' : "and state = 'open' and closed_at_local is null"}
+         order by updated_at_gh desc, number desc`)
+            .all(repository.id, normalizedLogin);
+        const latestRun = this.getLatestClusterRun(repository.id);
+        const strongestByThread = new Map();
+        if (latestRun && rows.length > 1) {
+            const edges = this.db
+                .prepare(`select
+              se.left_thread_id,
+              se.right_thread_id,
+              se.score,
+              t1.number as left_number,
+              t1.kind as left_kind,
+              t1.title as left_title,
+              t2.number as right_number,
+              t2.kind as right_kind,
+              t2.title as right_title
+           from similarity_edges se
+           join threads t1 on t1.id = se.left_thread_id
+           join threads t2 on t2.id = se.right_thread_id
+           where se.repo_id = ?
+             and se.cluster_run_id = ?
+             and lower(t1.author_login) = lower(?)
+             and lower(t2.author_login) = lower(?)
+             ${params.includeClosed ? '' : "and t1.state = 'open' and t1.closed_at_local is null and t2.state = 'open' and t2.closed_at_local is null"}`)
+                .all(repository.id, latestRun.id, normalizedLogin, normalizedLogin);
+            const updateStrongest = (sourceThreadId, match) => {
+                const previous = strongestByThread.get(sourceThreadId);
+                if (!previous || match.score > previous.score) {
+                    strongestByThread.set(sourceThreadId, match);
+                }
+            };
+            for (const edge of edges) {
+                updateStrongest(edge.left_thread_id, {
+                    threadId: edge.right_thread_id,
+                    number: edge.right_number,
+                    kind: edge.right_kind,
+                    title: edge.right_title,
+                    score: edge.score,
+                });
+                updateStrongest(edge.right_thread_id, {
+                    threadId: edge.left_thread_id,
+                    number: edge.left_number,
+                    kind: edge.left_kind,
+                    title: edge.left_title,
+                    score: edge.score,
+                });
+            }
+        }
+        return authorThreadsResponseSchema.parse({
+            repository,
+            authorLogin: normalizedLogin,
+            threads: rows.map((row) => ({
+                thread: threadToDto(row, clusterIds.get(row.id) ?? null),
+                strongestSameAuthorMatch: strongestByThread.get(row.id) ?? null,
+            })),
+        });
+    }
+    closeThreadLocally(params) {
+        const repository = this.requireRepository(params.owner, params.repo);
+        const row = this.db
+            .prepare('select * from threads where repo_id = ? and number = ? limit 1')
+            .get(repository.id, params.threadNumber);
+        if (!row) {
+            throw new Error(`Thread #${params.threadNumber} was not found for ${repository.fullName}.`);
+        }
+        const closedAt = nowIso();
+        this.db
+            .prepare(`update threads
+         set closed_at_local = ?,
+             close_reason_local = 'manual',
+             updated_at = ?
+         where id = ?`)
+            .run(closedAt, closedAt, row.id);
+        this.parsedEmbeddingCache.delete(repository.id);
+        const clusterIds = this.getLatestRunClusterIdsForThread(repository.id, row.id);
+        const clusterClosed = this.reconcileClusterCloseState(repository.id, clusterIds) > 0;
+        const updated = this.db.prepare('select * from threads where id = ? limit 1').get(row.id);
+        return closeResponseSchema.parse({
+            ok: true,
+            repository,
+            thread: threadToDto(updated),
+            clusterId: clusterIds[0] ?? null,
+            clusterClosed,
+            message: `Marked ${updated.kind} #${updated.number} closed locally.`,
+        });
+    }
+    closeClusterLocally(params) {
+        const repository = this.requireRepository(params.owner, params.repo);
+        const latestRun = this.getLatestClusterRun(repository.id);
+        if (!latestRun) {
+            throw new Error(`No completed cluster run found for ${repository.fullName}.`);
+        }
+        const row = this.db
+            .prepare('select id from clusters where repo_id = ? and cluster_run_id = ? and id = ? limit 1')
+            .get(repository.id, latestRun.id, params.clusterId);
+        if (!row) {
+            throw new Error(`Cluster ${params.clusterId} was not found for ${repository.fullName}.`);
+        }
+        const closedAt = nowIso();
+        this.db
+            .prepare(`update clusters
+         set closed_at_local = ?,
+             close_reason_local = 'manual'
+         where id = ?`)
+            .run(closedAt, row.id);
+        return closeResponseSchema.parse({
+            ok: true,
+            repository,
+            clusterId: row.id,
+            clusterClosed: true,
+            message: `Marked cluster ${row.id} closed locally.`,
         });
     }
     async syncRepository(params) {
@@ -366,6 +534,10 @@ export class GHCrawlService {
                 })
                 : 0;
             const threadsClosed = threadsClosedFromClosedSweep + threadsClosedFromDirectReconcile;
+            this.parsedEmbeddingCache.delete(repoId);
+            if (threadsClosed > 0) {
+                this.reconcileClusterCloseState(repoId);
+            }
             const finishedAt = nowIso();
             const reconciledOpenCloseAt = shouldSweepClosedOverlap || shouldReconcileMissingOpenThreads ? finishedAt : null;
             const nextSyncCursor = {
@@ -542,34 +714,21 @@ export class GHCrawlService {
                 title: meta.title,
             }));
             params.onProgress?.(`[cluster] loaded ${items.length} embedded thread(s) across ${new Set(rows.map((row) => row.source_kind)).size} source kind(s) for ${repository.fullName} k=${k} minScore=${minScore}`);
-            this.db.prepare('delete from cluster_members where cluster_id in (select id from clusters where cluster_run_id = ?)').run(runId);
-            this.db.prepare('delete from clusters where cluster_run_id = ?').run(runId);
-            this.db.prepare('delete from similarity_edges where cluster_run_id = ?').run(runId);
-            const aggregatedEdges = this.aggregateRepositoryEdges(rows, { limit: k, minScore });
+            const aggregatedEdges = this.aggregateRepositoryEdges(rows, {
+                limit: k,
+                minScore,
+                onProgress: params.onProgress,
+            });
             const edges = Array.from(aggregatedEdges.values()).map((entry) => ({
                 leftThreadId: entry.leftThreadId,
                 rightThreadId: entry.rightThreadId,
                 score: entry.score,
             }));
-            const insertEdge = this.db.prepare(`insert into similarity_edges (repo_id, cluster_run_id, left_thread_id, right_thread_id, method, score, explanation_json, created_at)
-         values (?, ?, ?, ?, ?, ?, ?, ?)`);
-            for (const edge of aggregatedEdges.values()) {
-                insertEdge.run(repository.id, runId, edge.leftThreadId, edge.rightThreadId, 'exact_cosine', edge.score, asJson({ sources: Array.from(edge.sourceKinds).sort(), model: this.config.embedModel }), nowIso());
-            }
             params.onProgress?.(`[cluster] built ${edges.length} similarity edge(s)`);
             const clusters = buildClusters(items.map((item) => ({ threadId: item.id, number: item.number, title: item.title })), edges);
-            const insertCluster = this.db.prepare('insert into clusters (repo_id, cluster_run_id, representative_thread_id, member_count, created_at) values (?, ?, ?, ?, ?)');
-            const insertMember = this.db.prepare('insert into cluster_members (cluster_id, thread_id, score_to_representative, created_at) values (?, ?, ?, ?)');
-            for (const cluster of clusters) {
-                const clusterResult = insertCluster.run(repository.id, runId, cluster.representativeThreadId, cluster.members.length, nowIso());
-                const clusterId = Number(clusterResult.lastInsertRowid);
-                for (const memberId of cluster.members) {
-                    const key = this.edgeKey(cluster.representativeThreadId, memberId);
-                    const score = memberId === cluster.representativeThreadId ? null : (aggregatedEdges.get(key)?.score ?? null);
-                    insertMember.run(clusterId, memberId, score, nowIso());
-                }
-            }
-            params.onProgress?.(`[cluster] persisted ${clusters.length} cluster(s)`);
+            this.persistClusterRun(repository.id, runId, aggregatedEdges, clusters);
+            this.pruneOldClusterRuns(repository.id, runId);
+            params.onProgress?.(`[cluster] persisted ${clusters.length} cluster(s) and pruned older cluster runs`);
             this.finishRun('cluster_runs', runId, 'completed', { edges: edges.length, clusters: clusters.length });
             return clusterResultSchema.parse({ runId, edges: edges.length, clusters: clusters.length });
         }
@@ -590,7 +749,7 @@ export class GHCrawlService {
            from documents_fts
            join documents d on d.id = documents_fts.rowid
            join threads t on t.id = d.thread_id
-           where t.repo_id = ? and t.state = 'open' and documents_fts match ?
+         where t.repo_id = ? and t.state = 'open' and t.closed_at_local is null and documents_fts match ?
            order by rank
            limit ?`)
                 .all(repository.id, params.query, limit * 2);
@@ -612,7 +771,7 @@ export class GHCrawlService {
         const threadRows = candidateIds.size
             ? this.db
                 .prepare(`select * from threads
-             where repo_id = ? and state = 'open' and id in (${[...candidateIds].map(() => '?').join(',')})
+             where repo_id = ? and state = 'open' and closed_at_local is null and id in (${[...candidateIds].map(() => '?').join(',')})
              order by updated_at_gh desc, number desc`)
                 .all(repository.id, ...candidateIds)
             : [];
@@ -724,7 +883,8 @@ export class GHCrawlService {
         }
         const rows = this.db
             .prepare(`select c.id, c.repo_id, c.representative_thread_id, c.member_count,
-                cm.thread_id, cm.score_to_representative, t.number, t.kind, t.title
+                c.closed_at_local, c.close_reason_local,
+                cm.thread_id, cm.score_to_representative, t.number, t.kind, t.title, t.state, t.closed_at_local as thread_closed_at_local
          from clusters c
          left join cluster_members cm on cm.cluster_id = c.id
          left join threads t on t.id = cm.thread_id
@@ -736,6 +896,9 @@ export class GHCrawlService {
             const cluster = clusters.get(row.id) ?? {
                 id: row.id,
                 repoId: row.repo_id,
+                isClosed: row.close_reason_local !== null,
+                closedAtLocal: row.closed_at_local,
+                closeReasonLocal: row.close_reason_local,
                 representativeThreadId: row.representative_thread_id,
                 memberCount: row.member_count,
                 members: [],
@@ -745,15 +908,20 @@ export class GHCrawlService {
                     threadId: row.thread_id,
                     number: row.number,
                     kind: row.kind,
+                    isClosed: row.state !== null && isEffectivelyClosed({ state: row.state, closed_at_local: row.thread_closed_at_local }),
                     title: row.title,
                     scoreToRepresentative: row.score_to_representative,
                 });
             }
             clusters.set(row.id, cluster);
         }
+        const clusterValues = Array.from(clusters.values()).map((cluster) => ({
+            ...cluster,
+            isClosed: cluster.isClosed || (cluster.memberCount > 0 && cluster.members.every((member) => member.isClosed)),
+        }));
         return clustersResponseSchema.parse({
             repository,
-            clusters: Array.from(clusters.values()),
+            clusters: clusterValues.filter((cluster) => (params.includeClosed ? true : !cluster.isClosed)),
         });
     }
     async refreshRepository(params) {
@@ -808,6 +976,7 @@ export class GHCrawlService {
             minSize: params.minSize,
             sort: params.sort,
             search: params.search,
+            includeClosedClusters: params.includeClosed === true,
         });
         const clusters = params.limit ? snapshot.clusters.slice(0, params.limit) : snapshot.clusters;
         return clusterSummariesResponseSchema.parse({
@@ -816,6 +985,9 @@ export class GHCrawlService {
             clusters: clusters.map((cluster) => ({
                 clusterId: cluster.clusterId,
                 displayTitle: cluster.displayTitle,
+                isClosed: cluster.isClosed,
+                closedAtLocal: cluster.closedAtLocal,
+                closeReasonLocal: cluster.closeReasonLocal,
                 totalCount: cluster.totalCount,
                 issueCount: cluster.issueCount,
                 pullRequestCount: cluster.pullRequestCount,
@@ -831,6 +1003,7 @@ export class GHCrawlService {
             owner: params.owner,
             repo: params.repo,
             minSize: 0,
+            includeClosedClusters: params.includeClosed === true,
         });
         const cluster = snapshot.clusters.find((item) => item.clusterId === params.clusterId);
         if (!cluster) {
@@ -863,6 +1036,9 @@ export class GHCrawlService {
             cluster: {
                 clusterId: cluster.clusterId,
                 displayTitle: cluster.displayTitle,
+                isClosed: cluster.isClosed,
+                closedAtLocal: cluster.closedAtLocal,
+                closeReasonLocal: cluster.closeReasonLocal,
                 totalCount: cluster.totalCount,
                 issueCount: cluster.issueCount,
                 pullRequestCount: cluster.pullRequestCount,
@@ -881,7 +1057,9 @@ export class GHCrawlService {
         if (!latestRun) {
             return { repository, stats, clusters: [] };
         }
+        const includeClosedClusters = params.includeClosedClusters ?? true;
         const clusters = this.listRawTuiClusters(repository.id, latestRun.id)
+            .filter((cluster) => (includeClosedClusters ? true : !cluster.isClosed))
             .filter((cluster) => cluster.totalCount >= (params.minSize ?? 10))
             .filter((cluster) => {
             const search = params.search?.trim().toLowerCase();
@@ -907,7 +1085,7 @@ export class GHCrawlService {
             throw new Error(`Cluster ${params.clusterId} was not found for ${repository.fullName}.`);
         }
         const rows = this.db
-            .prepare(`select t.id, t.number, t.kind, t.title, t.updated_at_gh, t.html_url, t.labels_json, cm.score_to_representative
+            .prepare(`select t.id, t.number, t.kind, t.state, t.closed_at_local, t.title, t.updated_at_gh, t.html_url, t.labels_json, cm.score_to_representative
          from cluster_members cm
          join threads t on t.id = cm.thread_id
          where cm.cluster_id = ?
@@ -919,6 +1097,9 @@ export class GHCrawlService {
         return {
             clusterId: summary.clusterId,
             displayTitle: summary.displayTitle,
+            isClosed: summary.isClosed,
+            closedAtLocal: summary.closedAtLocal,
+            closeReasonLocal: summary.closeReasonLocal,
             totalCount: summary.totalCount,
             issueCount: summary.issueCount,
             pullRequestCount: summary.pullRequestCount,
@@ -930,6 +1111,7 @@ export class GHCrawlService {
                 id: row.id,
                 number: row.number,
                 kind: row.kind,
+                isClosed: isEffectivelyClosed(row),
                 title: row.title,
                 updatedAtGh: row.updated_at_gh,
                 htmlUrl: row.html_url,
@@ -942,11 +1124,11 @@ export class GHCrawlService {
         const repository = this.requireRepository(params.owner, params.repo);
         const row = params.threadId
             ? (this.db
-                .prepare('select * from threads where repo_id = ? and id = ? and state = \'open\' limit 1')
+                .prepare('select * from threads where repo_id = ? and id = ? limit 1')
                 .get(repository.id, params.threadId) ?? null)
             : params.threadNumber
                 ? (this.db
-                    .prepare('select * from threads where repo_id = ? and number = ? and state = \'open\' limit 1')
+                    .prepare('select * from threads where repo_id = ? and number = ? limit 1')
                     .get(repository.id, params.threadNumber) ?? null)
                 : null;
         if (!row) {
@@ -979,17 +1161,20 @@ export class GHCrawlService {
         }
         let neighbors = [];
         if (params.includeNeighbors !== false) {
-            try {
-                neighbors = this.listNeighbors({
-                    owner: params.owner,
-                    repo: params.repo,
-                    threadNumber: row.number,
-                    limit: 8,
-                    minScore: 0.2,
-                }).neighbors;
-            }
-            catch {
-                neighbors = [];
+            neighbors = this.listStoredClusterNeighbors(repository.id, row.id, 8);
+            if (neighbors.length === 0) {
+                try {
+                    neighbors = this.listNeighbors({
+                        owner: params.owner,
+                        repo: params.repo,
+                        threadNumber: row.number,
+                        limit: 8,
+                        minScore: 0.2,
+                    }).neighbors;
+                }
+                catch {
+                    neighbors = [];
+                }
             }
         }
         return {
@@ -1103,7 +1288,7 @@ export class GHCrawlService {
         const counts = this.db
             .prepare(`select kind, count(*) as count
          from threads
-         where repo_id = ? and state = 'open'
+         where repo_id = ? and state = 'open' and closed_at_local is null
          group by kind`)
             .all(repoId);
         const latestRun = this.getLatestClusterRun(repoId);
@@ -1131,11 +1316,73 @@ export class GHCrawlService {
             .prepare("select id, finished_at from cluster_runs where repo_id = ? and status = 'completed' order by id desc limit 1")
             .get(repoId) ?? null);
     }
+    getLatestRunClusterIdsForThread(repoId, threadId) {
+        const latestRun = this.getLatestClusterRun(repoId);
+        if (!latestRun) {
+            return [];
+        }
+        return this.db
+            .prepare(`select cm.cluster_id
+           from cluster_members cm
+           join clusters c on c.id = cm.cluster_id
+           where c.repo_id = ? and c.cluster_run_id = ? and cm.thread_id = ?
+           order by cm.cluster_id asc`)
+            .all(repoId, latestRun.id, threadId).map((row) => row.cluster_id);
+    }
+    reconcileClusterCloseState(repoId, clusterIds) {
+        const latestRun = this.getLatestClusterRun(repoId);
+        if (!latestRun) {
+            return 0;
+        }
+        const resolvedClusterIds = clusterIds && clusterIds.length > 0
+            ? Array.from(new Set(clusterIds))
+            : this.db
+                .prepare('select id from clusters where repo_id = ? and cluster_run_id = ? order by id asc')
+                .all(repoId, latestRun.id).map((row) => row.id);
+        if (resolvedClusterIds.length === 0) {
+            return 0;
+        }
+        const summarize = this.db.prepare(`select
+          c.id,
+          c.close_reason_local,
+          count(*) as member_count,
+          sum(case when t.state != 'open' or t.closed_at_local is not null then 1 else 0 end) as closed_member_count
+       from clusters c
+       join cluster_members cm on cm.cluster_id = c.id
+       join threads t on t.id = cm.thread_id
+       where c.id = ?
+       group by c.id, c.close_reason_local`);
+        const markClosed = this.db.prepare(`update clusters
+       set closed_at_local = coalesce(closed_at_local, ?),
+           close_reason_local = 'all_members_closed'
+       where id = ?`);
+        const clearClosed = this.db.prepare(`update clusters
+       set closed_at_local = null,
+           close_reason_local = null
+       where id = ? and close_reason_local = 'all_members_closed'`);
+        let changed = 0;
+        for (const clusterId of resolvedClusterIds) {
+            const row = summarize.get(clusterId);
+            if (!row || row.close_reason_local === 'manual') {
+                continue;
+            }
+            if (row.member_count > 0 && row.closed_member_count >= row.member_count) {
+                const result = markClosed.run(nowIso(), clusterId);
+                changed += result.changes;
+                continue;
+            }
+            const cleared = clearClosed.run(clusterId);
+            changed += cleared.changes;
+        }
+        return changed;
+    }
     listRawTuiClusters(repoId, clusterRunId) {
         const rows = this.db
             .prepare(`select
             c.id as cluster_id,
             c.member_count,
+            c.closed_at_local,
+            c.close_reason_local,
             c.representative_thread_id,
             rt.number as representative_number,
             rt.kind as representative_kind,
@@ -1143,6 +1390,7 @@ export class GHCrawlService {
             max(coalesce(t.updated_at_gh, t.updated_at)) as latest_updated_at,
             sum(case when t.kind = 'issue' then 1 else 0 end) as issue_count,
             sum(case when t.kind = 'pull_request' then 1 else 0 end) as pull_request_count,
+            sum(case when t.state != 'open' or t.closed_at_local is not null then 1 else 0 end) as closed_member_count,
             group_concat(lower(coalesce(t.title, '')), ' ') as search_text
          from clusters c
          left join threads rt on rt.id = c.representative_thread_id
@@ -1160,6 +1408,9 @@ export class GHCrawlService {
         return rows.map((row) => ({
             clusterId: row.cluster_id,
             displayTitle: row.representative_title ?? `Cluster ${row.cluster_id}`,
+            isClosed: row.close_reason_local !== null || row.closed_member_count >= row.member_count,
+            closedAtLocal: row.closed_at_local,
+            closeReasonLocal: row.close_reason_local,
             totalCount: row.member_count,
             issueCount: row.issue_count,
             pullRequestCount: row.pull_request_count,
@@ -1297,6 +1548,7 @@ export class GHCrawlService {
          from threads
          where repo_id = ?
            and state = 'open'
+           and closed_at_local is null
            and (last_pulled_at is null or last_pulled_at < ?)
          order by number asc`)
             .all(params.repoId, params.crawlStartedAt);
@@ -1341,6 +1593,7 @@ export class GHCrawlService {
          from threads
          where repo_id = ?
            and state = 'open'
+           and closed_at_local is null
            and (last_pulled_at is null or last_pulled_at < ?)
          order by number asc`)
             .all(params.repoId, params.crawlStartedAt);
@@ -1628,11 +1881,12 @@ export class GHCrawlService {
     }
     loadStoredEmbeddings(repoId) {
         return this.db
-            .prepare(`select t.id, t.repo_id, t.number, t.kind, t.state, t.title, t.body, t.author_login, t.html_url, t.labels_json,
+            .prepare(`select t.id, t.repo_id, t.number, t.kind, t.state, t.closed_at_gh, t.closed_at_local, t.close_reason_local,
+                t.title, t.body, t.author_login, t.html_url, t.labels_json,
                 t.updated_at_gh, t.first_pulled_at, t.last_pulled_at, e.source_kind, e.embedding_json
          from threads t
          join document_embeddings e on e.thread_id = t.id
-         where t.repo_id = ? and t.state = 'open' and e.model = ?
+         where t.repo_id = ? and t.state = 'open' and t.closed_at_local is null and e.model = ?
          order by t.number asc, e.source_kind asc`)
             .all(repoId, this.config.embedModel);
     }
@@ -1648,10 +1902,55 @@ export class GHCrawlService {
         this.parsedEmbeddingCache.set(repoId, parsed);
         return parsed;
     }
+    listStoredClusterNeighbors(repoId, threadId, limit) {
+        const latestRun = this.getLatestClusterRun(repoId);
+        if (!latestRun) {
+            return [];
+        }
+        const rows = this.db
+            .prepare(`select
+            case
+              when se.left_thread_id = ? then se.right_thread_id
+              else se.left_thread_id
+            end as neighbor_thread_id,
+            case
+              when se.left_thread_id = ? then t2.number
+              else t1.number
+            end as neighbor_number,
+            case
+              when se.left_thread_id = ? then t2.kind
+              else t1.kind
+            end as neighbor_kind,
+            case
+              when se.left_thread_id = ? then t2.title
+              else t1.title
+            end as neighbor_title,
+            se.score
+         from similarity_edges se
+         join threads t1 on t1.id = se.left_thread_id
+         join threads t2 on t2.id = se.right_thread_id
+         where se.repo_id = ?
+           and se.cluster_run_id = ?
+           and (se.left_thread_id = ? or se.right_thread_id = ?)
+           and t1.state = 'open'
+           and t1.closed_at_local is null
+           and t2.state = 'open'
+           and t2.closed_at_local is null
+         order by se.score desc
+         limit ?`)
+            .all(threadId, threadId, threadId, threadId, repoId, latestRun.id, threadId, threadId, limit);
+        return rows.map((row) => ({
+            threadId: row.neighbor_thread_id,
+            number: row.neighbor_number,
+            kind: row.neighbor_kind,
+            title: row.neighbor_title,
+            score: row.score,
+        }));
+    }
     getEmbeddingWorkset(repoId, threadNumber) {
         let sql = `select t.id, t.number, t.title, t.body
        from threads t
-       where t.repo_id = ? and t.state = 'open'`;
+       where t.repo_id = ? and t.state = 'open' and t.closed_at_local is null`;
         const args = [repoId];
         if (threadNumber) {
             sql += ' and t.number = ?';
@@ -1684,7 +1983,7 @@ export class GHCrawlService {
         let sql = `select s.thread_id, s.summary_kind, s.summary_text
        from document_summaries s
        join threads t on t.id = s.thread_id
-       where t.repo_id = ? and t.state = 'open' and s.model = ?`;
+       where t.repo_id = ? and t.state = 'open' and t.closed_at_local is null and s.model = ?`;
         const args = [repoId, this.config.summaryModel];
         if (threadNumber) {
             sql += ' and t.number = ?';
@@ -1726,6 +2025,9 @@ export class GHCrawlService {
             bySource.set(row.source_kind, list);
         }
         const aggregated = new Map();
+        const totalItems = Array.from(bySource.values()).reduce((sum, items) => sum + items.length, 0);
+        let processedItems = 0;
+        let lastProgressAt = Date.now();
         for (const [sourceKind, items] of bySource.entries()) {
             for (const item of items) {
                 const neighbors = rankNearestNeighbors(items, {
@@ -1749,10 +2051,43 @@ export class GHCrawlService {
                         sourceKinds: new Set([sourceKind]),
                     });
                 }
+                processedItems += 1;
+                const now = Date.now();
+                if (params.onProgress && now - lastProgressAt >= CLUSTER_PROGRESS_INTERVAL_MS) {
+                    params.onProgress(`[cluster] identifying similarity edges ${processedItems}/${totalItems} source embeddings processed current_edges=${aggregated.size}`);
+                    lastProgressAt = now;
+                }
             }
         }
         return aggregated;
     }
+    persistClusterRun(repoId, runId, aggregatedEdges, clusters) {
+        const insertEdge = this.db.prepare(`insert into similarity_edges (repo_id, cluster_run_id, left_thread_id, right_thread_id, method, score, explanation_json, created_at)
+       values (?, ?, ?, ?, ?, ?, ?, ?)`);
+        const insertCluster = this.db.prepare('insert into clusters (repo_id, cluster_run_id, representative_thread_id, member_count, created_at) values (?, ?, ?, ?, ?)');
+        const insertMember = this.db.prepare('insert into cluster_members (cluster_id, thread_id, score_to_representative, created_at) values (?, ?, ?, ?)');
+        this.db.transaction(() => {
+            this.db.prepare('delete from cluster_members where cluster_id in (select id from clusters where cluster_run_id = ?)').run(runId);
+            this.db.prepare('delete from clusters where cluster_run_id = ?').run(runId);
+            this.db.prepare('delete from similarity_edges where cluster_run_id = ?').run(runId);
+            const createdAt = nowIso();
+            for (const edge of aggregatedEdges.values()) {
+                insertEdge.run(repoId, runId, edge.leftThreadId, edge.rightThreadId, 'exact_cosine', edge.score, asJson({ sources: Array.from(edge.sourceKinds).sort(), model: this.config.embedModel }), createdAt);
+            }
+            for (const cluster of clusters) {
+                const clusterResult = insertCluster.run(repoId, runId, cluster.representativeThreadId, cluster.members.length, createdAt);
+                const clusterId = Number(clusterResult.lastInsertRowid);
+                for (const memberId of cluster.members) {
+                    const key = this.edgeKey(cluster.representativeThreadId, memberId);
+                    const score = memberId === cluster.representativeThreadId ? null : (aggregatedEdges.get(key)?.score ?? null);
+                    insertMember.run(clusterId, memberId, score, createdAt);
+                }
+            }
+        })();
+    }
+    pruneOldClusterRuns(repoId, keepRunId) {
+        this.db.prepare('delete from cluster_runs where repo_id = ? and id <> ?').run(repoId, keepRunId);
+    }
     upsertSummary(threadId, contentHash, summaryKind, summaryText) {
         this.db
             .prepare(`insert into document_summaries (thread_id, summary_kind, model, content_hash, summary_text, created_at, updated_at)