@ghcrawl/api-core 0.3.0 → 0.4.0

package/dist/service.js

@@ -1,7 +1,7 @@
 import http from 'node:http';
 import crypto from 'node:crypto';
 import { IterableMapper } from '@shutterstock/p-map-iterable';
-import { actionResponseSchema, clusterDetailResponseSchema, clusterResultSchema, clusterSummariesResponseSchema, clustersResponseSchema, embedResultSchema, healthResponseSchema, neighborsResponseSchema, refreshResponseSchema, repositoriesResponseSchema, searchResponseSchema, syncResultSchema, threadsResponseSchema, } from '@ghcrawl/api-contract';
+import { actionResponseSchema, authorThreadsResponseSchema, closeResponseSchema, clusterDetailResponseSchema, clusterResultSchema, clusterSummariesResponseSchema, clustersResponseSchema, embedResultSchema, healthResponseSchema, neighborsResponseSchema, refreshResponseSchema, repositoriesResponseSchema, searchResponseSchema, syncResultSchema, threadsResponseSchema, } from '@ghcrawl/api-contract';
 import { buildClusters } from './cluster/build.js';
 import { ensureRuntimeDirs, isLikelyGitHubToken, isLikelyOpenAiApiKey, loadConfig, requireGithubToken, requireOpenAiKey, } from './config.js';
 import { migrate } from './db/migrate.js';
@@ -27,6 +27,9 @@ function parseIso(value) {
     const parsed = Date.parse(value);
     return Number.isNaN(parsed) ? null : parsed;
 }
+function isEffectivelyClosed(row) {
+    return row.state !== 'open' || row.closed_at_local !== null;
+}
 function isMissingGitHubResourceError(error) {
     const status = typeof error?.status === 'number' ? Number(error.status) : null;
     if (status === 404 || status === 410) {
@@ -150,6 +153,10 @@ function threadToDto(row, clusterId) {
         number: row.number,
         kind: row.kind,
         state: row.state,
+        isClosed: isEffectivelyClosed(row),
+        closedAtGh: row.closed_at_gh ?? null,
+        closedAtLocal: row.closed_at_local ?? null,
+        closeReasonLocal: row.close_reason_local ?? null,
         title: row.title,
         body: row.body,
         authorLogin: row.author_login,
@@ -260,8 +267,11 @@ export class GHCrawlService {
             .all(repository.id, repository.id);
         for (const row of clusterRows)
             clusterIds.set(row.thread_id, row.cluster_id);
-        let sql = "select * from threads where repo_id = ? and state = 'open'";
+        let sql = 'select * from threads where repo_id = ?';
         const args = [repository.id];
+        if (!params.includeClosed) {
+            sql += " and state = 'open' and closed_at_local is null";
+        }
         if (params.kind) {
             sql += ' and kind = ?';
             args.push(params.kind);
@@ -291,6 +301,145 @@ export class GHCrawlService {
             threads: orderedRows.map((row) => threadToDto(row, clusterIds.get(row.id) ?? null)),
         });
     }
+    listAuthorThreads(params) {
+        const repository = this.requireRepository(params.owner, params.repo);
+        const normalizedLogin = params.login.trim();
+        if (!normalizedLogin) {
+            return authorThreadsResponseSchema.parse({
+                repository,
+                authorLogin: '',
+                threads: [],
+            });
+        }
+        const clusterIds = new Map();
+        const clusterRows = this.db
+            .prepare(`select cm.thread_id, cm.cluster_id
+         from cluster_members cm
+         join clusters c on c.id = cm.cluster_id
+         where c.repo_id = ? and c.cluster_run_id = (
+           select id from cluster_runs where repo_id = ? and status = 'completed' order by id desc limit 1
+         )`)
+            .all(repository.id, repository.id);
+        for (const row of clusterRows)
+            clusterIds.set(row.thread_id, row.cluster_id);
+        const rows = this.db
+            .prepare(`select *
+         from threads
+         where repo_id = ? and lower(author_login) = lower(?)
+           ${params.includeClosed ? '' : "and state = 'open' and closed_at_local is null"}
+         order by updated_at_gh desc, number desc`)
+            .all(repository.id, normalizedLogin);
+        const latestRun = this.getLatestClusterRun(repository.id);
+        const strongestByThread = new Map();
+        if (latestRun && rows.length > 1) {
+            const edges = this.db
+                .prepare(`select
+              se.left_thread_id,
+              se.right_thread_id,
+              se.score,
+              t1.number as left_number,
+              t1.kind as left_kind,
+              t1.title as left_title,
+              t2.number as right_number,
+              t2.kind as right_kind,
+              t2.title as right_title
+           from similarity_edges se
+           join threads t1 on t1.id = se.left_thread_id
+           join threads t2 on t2.id = se.right_thread_id
+           where se.repo_id = ?
+             and se.cluster_run_id = ?
+             and lower(t1.author_login) = lower(?)
+             and lower(t2.author_login) = lower(?)
+             ${params.includeClosed ? '' : "and t1.state = 'open' and t1.closed_at_local is null and t2.state = 'open' and t2.closed_at_local is null"}`)
+                .all(repository.id, latestRun.id, normalizedLogin, normalizedLogin);
+            const updateStrongest = (sourceThreadId, match) => {
+                const previous = strongestByThread.get(sourceThreadId);
+                if (!previous || match.score > previous.score) {
+                    strongestByThread.set(sourceThreadId, match);
+                }
+            };
+            for (const edge of edges) {
+                updateStrongest(edge.left_thread_id, {
+                    threadId: edge.right_thread_id,
+                    number: edge.right_number,
+                    kind: edge.right_kind,
+                    title: edge.right_title,
+                    score: edge.score,
+                });
+                updateStrongest(edge.right_thread_id, {
+                    threadId: edge.left_thread_id,
+                    number: edge.left_number,
+                    kind: edge.left_kind,
+                    title: edge.left_title,
+                    score: edge.score,
+                });
+            }
+        }
+        return authorThreadsResponseSchema.parse({
+            repository,
+            authorLogin: normalizedLogin,
+            threads: rows.map((row) => ({
+                thread: threadToDto(row, clusterIds.get(row.id) ?? null),
+                strongestSameAuthorMatch: strongestByThread.get(row.id) ?? null,
+            })),
+        });
+    }
+    closeThreadLocally(params) {
+        const repository = this.requireRepository(params.owner, params.repo);
+        const row = this.db
+            .prepare('select * from threads where repo_id = ? and number = ? limit 1')
+            .get(repository.id, params.threadNumber);
+        if (!row) {
+            throw new Error(`Thread #${params.threadNumber} was not found for ${repository.fullName}.`);
+        }
+        const closedAt = nowIso();
+        this.db
+            .prepare(`update threads
+         set closed_at_local = ?,
+             close_reason_local = 'manual',
+             updated_at = ?
+         where id = ?`)
+            .run(closedAt, closedAt, row.id);
+        this.parsedEmbeddingCache.delete(repository.id);
+        const clusterIds = this.getLatestRunClusterIdsForThread(repository.id, row.id);
+        const clusterClosed = this.reconcileClusterCloseState(repository.id, clusterIds) > 0;
+        const updated = this.db.prepare('select * from threads where id = ? limit 1').get(row.id);
+        return closeResponseSchema.parse({
+            ok: true,
+            repository,
+            thread: threadToDto(updated),
+            clusterId: clusterIds[0] ?? null,
+            clusterClosed,
+            message: `Marked ${updated.kind} #${updated.number} closed locally.`,
+        });
+    }
+    closeClusterLocally(params) {
+        const repository = this.requireRepository(params.owner, params.repo);
+        const latestRun = this.getLatestClusterRun(repository.id);
+        if (!latestRun) {
+            throw new Error(`No completed cluster run found for ${repository.fullName}.`);
+        }
+        const row = this.db
+            .prepare('select id from clusters where repo_id = ? and cluster_run_id = ? and id = ? limit 1')
+            .get(repository.id, latestRun.id, params.clusterId);
+        if (!row) {
+            throw new Error(`Cluster ${params.clusterId} was not found for ${repository.fullName}.`);
+        }
+        const closedAt = nowIso();
+        this.db
+            .prepare(`update clusters
+         set closed_at_local = ?,
+             close_reason_local = 'manual'
+         where id = ?`)
+            .run(closedAt, row.id);
+        return closeResponseSchema.parse({
+            ok: true,
+            repository,
+            clusterId: row.id,
+            clusterClosed: true,
+            message: `Marked cluster ${row.id} closed locally.`,
+        });
+    }
     async syncRepository(params) {
         const crawlStartedAt = params.startedAt ?? nowIso();
         const includeComments = params.includeComments ?? false;
@@ -385,6 +534,10 @@ export class GHCrawlService {
                 })
                 : 0;
             const threadsClosed = threadsClosedFromClosedSweep + threadsClosedFromDirectReconcile;
+            this.parsedEmbeddingCache.delete(repoId);
+            if (threadsClosed > 0) {
+                this.reconcileClusterCloseState(repoId);
+            }
             const finishedAt = nowIso();
             const reconciledOpenCloseAt = shouldSweepClosedOverlap || shouldReconcileMissingOpenThreads ? finishedAt : null;
             const nextSyncCursor = {
@@ -596,7 +749,7 @@ export class GHCrawlService {
            from documents_fts
            join documents d on d.id = documents_fts.rowid
            join threads t on t.id = d.thread_id
-           where t.repo_id = ? and t.state = 'open' and documents_fts match ?
+         where t.repo_id = ? and t.state = 'open' and t.closed_at_local is null and documents_fts match ?
            order by rank
            limit ?`)
                 .all(repository.id, params.query, limit * 2);
@@ -618,7 +771,7 @@ export class GHCrawlService {
         const threadRows = candidateIds.size
             ? this.db
                 .prepare(`select * from threads
-             where repo_id = ? and state = 'open' and id in (${[...candidateIds].map(() => '?').join(',')})
+             where repo_id = ? and state = 'open' and closed_at_local is null and id in (${[...candidateIds].map(() => '?').join(',')})
              order by updated_at_gh desc, number desc`)
                 .all(repository.id, ...candidateIds)
             : [];
@@ -730,7 +883,8 @@ export class GHCrawlService {
         }
         const rows = this.db
             .prepare(`select c.id, c.repo_id, c.representative_thread_id, c.member_count,
-                cm.thread_id, cm.score_to_representative, t.number, t.kind, t.title
+                c.closed_at_local, c.close_reason_local,
+                cm.thread_id, cm.score_to_representative, t.number, t.kind, t.title, t.state, t.closed_at_local as thread_closed_at_local
          from clusters c
          left join cluster_members cm on cm.cluster_id = c.id
          left join threads t on t.id = cm.thread_id
@@ -742,6 +896,9 @@ export class GHCrawlService {
             const cluster = clusters.get(row.id) ?? {
                 id: row.id,
                 repoId: row.repo_id,
+                isClosed: row.close_reason_local !== null,
+                closedAtLocal: row.closed_at_local,
+                closeReasonLocal: row.close_reason_local,
                 representativeThreadId: row.representative_thread_id,
                 memberCount: row.member_count,
                 members: [],
@@ -751,15 +908,20 @@ export class GHCrawlService {
                     threadId: row.thread_id,
                     number: row.number,
                     kind: row.kind,
+                    isClosed: row.state !== null && isEffectivelyClosed({ state: row.state, closed_at_local: row.thread_closed_at_local }),
                     title: row.title,
                     scoreToRepresentative: row.score_to_representative,
                 });
             }
             clusters.set(row.id, cluster);
         }
+        const clusterValues = Array.from(clusters.values()).map((cluster) => ({
+            ...cluster,
+            isClosed: cluster.isClosed || (cluster.memberCount > 0 && cluster.members.every((member) => member.isClosed)),
+        }));
         return clustersResponseSchema.parse({
             repository,
-            clusters: Array.from(clusters.values()),
+            clusters: clusterValues.filter((cluster) => (params.includeClosed ? true : !cluster.isClosed)),
         });
     }
     async refreshRepository(params) {
@@ -814,6 +976,7 @@ export class GHCrawlService {
             minSize: params.minSize,
             sort: params.sort,
             search: params.search,
+            includeClosedClusters: params.includeClosed === true,
         });
         const clusters = params.limit ? snapshot.clusters.slice(0, params.limit) : snapshot.clusters;
         return clusterSummariesResponseSchema.parse({
@@ -822,6 +985,9 @@ export class GHCrawlService {
             clusters: clusters.map((cluster) => ({
                 clusterId: cluster.clusterId,
                 displayTitle: cluster.displayTitle,
+                isClosed: cluster.isClosed,
+                closedAtLocal: cluster.closedAtLocal,
+                closeReasonLocal: cluster.closeReasonLocal,
                 totalCount: cluster.totalCount,
                 issueCount: cluster.issueCount,
                 pullRequestCount: cluster.pullRequestCount,
@@ -837,6 +1003,7 @@ export class GHCrawlService {
             owner: params.owner,
             repo: params.repo,
             minSize: 0,
+            includeClosedClusters: params.includeClosed === true,
         });
         const cluster = snapshot.clusters.find((item) => item.clusterId === params.clusterId);
         if (!cluster) {
@@ -869,6 +1036,9 @@ export class GHCrawlService {
             cluster: {
                 clusterId: cluster.clusterId,
                 displayTitle: cluster.displayTitle,
+                isClosed: cluster.isClosed,
+                closedAtLocal: cluster.closedAtLocal,
+                closeReasonLocal: cluster.closeReasonLocal,
                 totalCount: cluster.totalCount,
                 issueCount: cluster.issueCount,
                 pullRequestCount: cluster.pullRequestCount,
@@ -887,7 +1057,9 @@ export class GHCrawlService {
         if (!latestRun) {
             return { repository, stats, clusters: [] };
         }
+        const includeClosedClusters = params.includeClosedClusters ?? true;
         const clusters = this.listRawTuiClusters(repository.id, latestRun.id)
+            .filter((cluster) => (includeClosedClusters ? true : !cluster.isClosed))
             .filter((cluster) => cluster.totalCount >= (params.minSize ?? 10))
             .filter((cluster) => {
             const search = params.search?.trim().toLowerCase();
@@ -913,7 +1085,7 @@ export class GHCrawlService {
             throw new Error(`Cluster ${params.clusterId} was not found for ${repository.fullName}.`);
         }
         const rows = this.db
-            .prepare(`select t.id, t.number, t.kind, t.title, t.updated_at_gh, t.html_url, t.labels_json, cm.score_to_representative
+            .prepare(`select t.id, t.number, t.kind, t.state, t.closed_at_local, t.title, t.updated_at_gh, t.html_url, t.labels_json, cm.score_to_representative
          from cluster_members cm
          join threads t on t.id = cm.thread_id
          where cm.cluster_id = ?
@@ -925,6 +1097,9 @@ export class GHCrawlService {
         return {
             clusterId: summary.clusterId,
             displayTitle: summary.displayTitle,
+            isClosed: summary.isClosed,
+            closedAtLocal: summary.closedAtLocal,
+            closeReasonLocal: summary.closeReasonLocal,
             totalCount: summary.totalCount,
             issueCount: summary.issueCount,
             pullRequestCount: summary.pullRequestCount,
@@ -936,6 +1111,7 @@ export class GHCrawlService {
                 id: row.id,
                 number: row.number,
                 kind: row.kind,
+                isClosed: isEffectivelyClosed(row),
                 title: row.title,
                 updatedAtGh: row.updated_at_gh,
                 htmlUrl: row.html_url,
@@ -948,11 +1124,11 @@ export class GHCrawlService {
         const repository = this.requireRepository(params.owner, params.repo);
         const row = params.threadId
             ? (this.db
-                .prepare('select * from threads where repo_id = ? and id = ? and state = \'open\' limit 1')
+                .prepare('select * from threads where repo_id = ? and id = ? limit 1')
                 .get(repository.id, params.threadId) ?? null)
             : params.threadNumber
                 ? (this.db
-                    .prepare('select * from threads where repo_id = ? and number = ? and state = \'open\' limit 1')
+                    .prepare('select * from threads where repo_id = ? and number = ? limit 1')
                     .get(repository.id, params.threadNumber) ?? null)
                 : null;
         if (!row) {
@@ -1112,7 +1288,7 @@ export class GHCrawlService {
         const counts = this.db
             .prepare(`select kind, count(*) as count
          from threads
-         where repo_id = ? and state = 'open'
+         where repo_id = ? and state = 'open' and closed_at_local is null
          group by kind`)
             .all(repoId);
         const latestRun = this.getLatestClusterRun(repoId);
@@ -1140,11 +1316,73 @@ export class GHCrawlService {
             .prepare("select id, finished_at from cluster_runs where repo_id = ? and status = 'completed' order by id desc limit 1")
             .get(repoId) ?? null);
     }
+    getLatestRunClusterIdsForThread(repoId, threadId) {
+        const latestRun = this.getLatestClusterRun(repoId);
+        if (!latestRun) {
+            return [];
+        }
+        return this.db
+            .prepare(`select cm.cluster_id
+           from cluster_members cm
+           join clusters c on c.id = cm.cluster_id
+           where c.repo_id = ? and c.cluster_run_id = ? and cm.thread_id = ?
+           order by cm.cluster_id asc`)
+            .all(repoId, latestRun.id, threadId).map((row) => row.cluster_id);
+    }
+    reconcileClusterCloseState(repoId, clusterIds) {
+        const latestRun = this.getLatestClusterRun(repoId);
+        if (!latestRun) {
+            return 0;
+        }
+        const resolvedClusterIds = clusterIds && clusterIds.length > 0
+            ? Array.from(new Set(clusterIds))
+            : this.db
+                .prepare('select id from clusters where repo_id = ? and cluster_run_id = ? order by id asc')
+                .all(repoId, latestRun.id).map((row) => row.id);
+        if (resolvedClusterIds.length === 0) {
+            return 0;
+        }
+        const summarize = this.db.prepare(`select
+          c.id,
+          c.close_reason_local,
+          count(*) as member_count,
+          sum(case when t.state != 'open' or t.closed_at_local is not null then 1 else 0 end) as closed_member_count
+       from clusters c
+       join cluster_members cm on cm.cluster_id = c.id
+       join threads t on t.id = cm.thread_id
+       where c.id = ?
+       group by c.id, c.close_reason_local`);
+        const markClosed = this.db.prepare(`update clusters
+       set closed_at_local = coalesce(closed_at_local, ?),
+           close_reason_local = 'all_members_closed'
+       where id = ?`);
+        const clearClosed = this.db.prepare(`update clusters
+       set closed_at_local = null,
+           close_reason_local = null
+       where id = ? and close_reason_local = 'all_members_closed'`);
+        let changed = 0;
+        for (const clusterId of resolvedClusterIds) {
+            const row = summarize.get(clusterId);
+            if (!row || row.close_reason_local === 'manual') {
+                continue;
+            }
+            if (row.member_count > 0 && row.closed_member_count >= row.member_count) {
+                const result = markClosed.run(nowIso(), clusterId);
+                changed += result.changes;
+                continue;
+            }
+            const cleared = clearClosed.run(clusterId);
+            changed += cleared.changes;
+        }
+        return changed;
+    }
     listRawTuiClusters(repoId, clusterRunId) {
         const rows = this.db
             .prepare(`select
             c.id as cluster_id,
             c.member_count,
+            c.closed_at_local,
+            c.close_reason_local,
             c.representative_thread_id,
             rt.number as representative_number,
             rt.kind as representative_kind,
@@ -1152,6 +1390,7 @@ export class GHCrawlService {
             max(coalesce(t.updated_at_gh, t.updated_at)) as latest_updated_at,
             sum(case when t.kind = 'issue' then 1 else 0 end) as issue_count,
             sum(case when t.kind = 'pull_request' then 1 else 0 end) as pull_request_count,
+            sum(case when t.state != 'open' or t.closed_at_local is not null then 1 else 0 end) as closed_member_count,
             group_concat(lower(coalesce(t.title, '')), ' ') as search_text
          from clusters c
          left join threads rt on rt.id = c.representative_thread_id
@@ -1169,6 +1408,9 @@ export class GHCrawlService {
         return rows.map((row) => ({
             clusterId: row.cluster_id,
             displayTitle: row.representative_title ?? `Cluster ${row.cluster_id}`,
+            isClosed: row.close_reason_local !== null || row.closed_member_count >= row.member_count,
+            closedAtLocal: row.closed_at_local,
+            closeReasonLocal: row.close_reason_local,
             totalCount: row.member_count,
             issueCount: row.issue_count,
             pullRequestCount: row.pull_request_count,
@@ -1306,6 +1548,7 @@ export class GHCrawlService {
          from threads
          where repo_id = ?
            and state = 'open'
+           and closed_at_local is null
            and (last_pulled_at is null or last_pulled_at < ?)
          order by number asc`)
             .all(params.repoId, params.crawlStartedAt);
@@ -1350,6 +1593,7 @@ export class GHCrawlService {
          from threads
          where repo_id = ?
            and state = 'open'
+           and closed_at_local is null
            and (last_pulled_at is null or last_pulled_at < ?)
          order by number asc`)
             .all(params.repoId, params.crawlStartedAt);
@@ -1637,11 +1881,12 @@ export class GHCrawlService {
     }
     loadStoredEmbeddings(repoId) {
         return this.db
-            .prepare(`select t.id, t.repo_id, t.number, t.kind, t.state, t.title, t.body, t.author_login, t.html_url, t.labels_json,
+            .prepare(`select t.id, t.repo_id, t.number, t.kind, t.state, t.closed_at_gh, t.closed_at_local, t.close_reason_local,
+                t.title, t.body, t.author_login, t.html_url, t.labels_json,
                 t.updated_at_gh, t.first_pulled_at, t.last_pulled_at, e.source_kind, e.embedding_json
          from threads t
          join document_embeddings e on e.thread_id = t.id
-         where t.repo_id = ? and t.state = 'open' and e.model = ?
+         where t.repo_id = ? and t.state = 'open' and t.closed_at_local is null and e.model = ?
          order by t.number asc, e.source_kind asc`)
             .all(repoId, this.config.embedModel);
     }
@@ -1688,7 +1933,9 @@ export class GHCrawlService {
            and se.cluster_run_id = ?
            and (se.left_thread_id = ? or se.right_thread_id = ?)
            and t1.state = 'open'
+           and t1.closed_at_local is null
            and t2.state = 'open'
+           and t2.closed_at_local is null
          order by se.score desc
          limit ?`)
             .all(threadId, threadId, threadId, threadId, repoId, latestRun.id, threadId, threadId, limit);
@@ -1703,7 +1950,7 @@ export class GHCrawlService {
     getEmbeddingWorkset(repoId, threadNumber) {
         let sql = `select t.id, t.number, t.title, t.body
        from threads t
-       where t.repo_id = ? and t.state = 'open'`;
+       where t.repo_id = ? and t.state = 'open' and t.closed_at_local is null`;
         const args = [repoId];
         if (threadNumber) {
             sql += ' and t.number = ?';
@@ -1736,7 +1983,7 @@ export class GHCrawlService {
         let sql = `select s.thread_id, s.summary_kind, s.summary_text
        from document_summaries s
        join threads t on t.id = s.thread_id
-       where t.repo_id = ? and t.state = 'open' and s.model = ?`;
+       where t.repo_id = ? and t.state = 'open' and t.closed_at_local is null and s.model = ?`;
         const args = [repoId, this.config.summaryModel];
         if (threadNumber) {
             sql += ' and t.number = ?';