@ghcrawl/api-core 0.1.0

package/src/db/migrate.ts

@@ -0,0 +1,235 @@
+import type { SqliteDatabase } from './sqlite.js';
+
+const migrationStatements = [
+  `
+  create table if not exists repositories (
+    id integer primary key,
+    owner text not null,
+    name text not null,
+    full_name text not null unique,
+    github_repo_id text,
+    raw_json text not null,
+    updated_at text not null
+  )
+  `,
+  `
+  create table if not exists threads (
+    id integer primary key,
+    repo_id integer not null references repositories(id) on delete cascade,
+    github_id text not null,
+    number integer not null,
+    kind text not null,
+    state text not null,
+    title text not null,
+    body text,
+    author_login text,
+    author_type text,
+    html_url text not null,
+    labels_json text not null,
+    assignees_json text not null,
+    raw_json text not null,
+    content_hash text not null,
+    is_draft integer not null default 0,
+    created_at_gh text,
+    updated_at_gh text,
+    closed_at_gh text,
+    merged_at_gh text,
+    first_pulled_at text,
+    last_pulled_at text,
+    updated_at text not null,
+    unique(repo_id, kind, number)
+  )
+  `,
+  `
+  create table if not exists comments (
+    id integer primary key,
+    thread_id integer not null references threads(id) on delete cascade,
+    github_id text not null,
+    comment_type text not null,
+    author_login text,
+    author_type text,
+    body text not null,
+    is_bot integer not null default 0,
+    raw_json text not null,
+    created_at_gh text,
+    updated_at_gh text,
+    unique(thread_id, comment_type, github_id)
+  )
+  `,
+  `
+  create table if not exists documents (
+    id integer primary key,
+    thread_id integer not null unique references threads(id) on delete cascade,
+    title text not null,
+    body text,
+    raw_text text not null,
+    dedupe_text text not null,
+    updated_at text not null
+  )
+  `,
+  `
+  create virtual table if not exists documents_fts using fts5(
+    title,
+    body,
+    raw_text,
+    dedupe_text,
+    content='documents',
+    content_rowid='id'
+  )
+  `,
+  `
+  create trigger if not exists documents_ai after insert on documents begin
+    insert into documents_fts(rowid, title, body, raw_text, dedupe_text)
+    values (new.id, new.title, new.body, new.raw_text, new.dedupe_text);
+  end
+  `,
+  `
+  create trigger if not exists documents_ad after delete on documents begin
+    insert into documents_fts(documents_fts, rowid, title, body, raw_text, dedupe_text)
+    values ('delete', old.id, old.title, old.body, old.raw_text, old.dedupe_text);
+  end
+  `,
+  `
+  create trigger if not exists documents_au after update on documents begin
+    insert into documents_fts(documents_fts, rowid, title, body, raw_text, dedupe_text)
+    values ('delete', old.id, old.title, old.body, old.raw_text, old.dedupe_text);
+    insert into documents_fts(rowid, title, body, raw_text, dedupe_text)
+    values (new.id, new.title, new.body, new.raw_text, new.dedupe_text);
+  end
+  `,
+  `
+  create table if not exists document_summaries (
+    id integer primary key,
+    thread_id integer not null references threads(id) on delete cascade,
+    summary_kind text not null,
+    model text not null,
+    content_hash text not null,
+    summary_text text not null,
+    created_at text not null,
+    updated_at text not null,
+    unique(thread_id, summary_kind, model)
+  )
+  `,
+  `
+  create table if not exists document_embeddings (
+    id integer primary key,
+    thread_id integer not null references threads(id) on delete cascade,
+    source_kind text not null,
+    model text not null,
+    dimensions integer not null,
+    content_hash text not null,
+    embedding_json text not null,
+    created_at text not null,
+    updated_at text not null,
+    unique(thread_id, source_kind, model)
+  )
+  `,
+  `
+  create table if not exists sync_runs (
+    id integer primary key,
+    repo_id integer references repositories(id) on delete cascade,
+    scope text not null,
+    status text not null,
+    started_at text not null,
+    finished_at text,
+    stats_json text,
+    error_text text
+  )
+  `,
+  `
+  create table if not exists repo_sync_state (
+    repo_id integer primary key references repositories(id) on delete cascade,
+    last_full_open_scan_started_at text,
+    last_overlapping_open_scan_completed_at text,
+    last_non_overlapping_scan_completed_at text,
+    last_open_close_reconciled_at text,
+    updated_at text not null
+  )
+  `,
+  `
+  create table if not exists summary_runs (
+    id integer primary key,
+    repo_id integer references repositories(id) on delete cascade,
+    scope text not null,
+    status text not null,
+    started_at text not null,
+    finished_at text,
+    stats_json text,
+    error_text text
+  )
+  `,
+  `
+  create table if not exists embedding_runs (
+    id integer primary key,
+    repo_id integer references repositories(id) on delete cascade,
+    scope text not null,
+    status text not null,
+    started_at text not null,
+    finished_at text,
+    stats_json text,
+    error_text text
+  )
+  `,
+  `
+  create table if not exists cluster_runs (
+    id integer primary key,
+    repo_id integer references repositories(id) on delete cascade,
+    scope text not null,
+    status text not null,
+    started_at text not null,
+    finished_at text,
+    stats_json text,
+    error_text text
+  )
+  `,
+  `
+  create table if not exists similarity_edges (
+    id integer primary key,
+    repo_id integer not null references repositories(id) on delete cascade,
+    cluster_run_id integer references cluster_runs(id) on delete cascade,
+    left_thread_id integer not null references threads(id) on delete cascade,
+    right_thread_id integer not null references threads(id) on delete cascade,
+    method text not null,
+    score real not null,
+    explanation_json text not null,
+    created_at text not null,
+    unique(cluster_run_id, left_thread_id, right_thread_id)
+  )
+  `,
+  `
+  create table if not exists clusters (
+    id integer primary key,
+    repo_id integer not null references repositories(id) on delete cascade,
+    cluster_run_id integer not null references cluster_runs(id) on delete cascade,
+    representative_thread_id integer references threads(id) on delete set null,
+    member_count integer not null,
+    created_at text not null
+  )
+  `,
+  `
+  create table if not exists cluster_members (
+    cluster_id integer not null references clusters(id) on delete cascade,
+    thread_id integer not null references threads(id) on delete cascade,
+    score_to_representative real,
+    created_at text not null,
+    primary key (cluster_id, thread_id)
+  )
+  `
+];
+
+export function migrate(db: SqliteDatabase): void {
+  for (const statement of migrationStatements) {
+    db.exec(statement);
+  }
+
+  const threadColumns = new Set(
+    (db.prepare('pragma table_info(threads)').all() as Array<{ name: string }>).map((column) => column.name),
+  );
+
+  if (!threadColumns.has('first_pulled_at')) {
+    db.exec('alter table threads add column first_pulled_at text');
+  }
+  if (!threadColumns.has('last_pulled_at')) {
+    db.exec('alter table threads add column last_pulled_at text');
+  }
+}

package/src/db/sqlite.ts

@@ -0,0 +1,14 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+import BetterSqlite3 from 'better-sqlite3';
+
+export type SqliteDatabase = InstanceType<typeof BetterSqlite3>;
+
+export function openDb(dbPath: string): SqliteDatabase {
+  fs.mkdirSync(path.dirname(dbPath), { recursive: true });
+  const db = new BetterSqlite3(dbPath);
+  db.pragma('journal_mode = WAL');
+  db.pragma('foreign_keys = ON');
+  return db;
+}

package/src/documents/normalize.test.ts

@@ -0,0 +1,25 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+import { buildCanonicalDocument, isBotLikeAuthor } from './normalize.js';
+
+test('bot detection catches bot users and common automation', () => {
+  assert.equal(isBotLikeAuthor({ authorLogin: 'dependabot[bot]' }), true);
+  assert.equal(isBotLikeAuthor({ authorType: 'Bot' }), true);
+  assert.equal(isBotLikeAuthor({ authorLogin: 'maintainer' }), false);
+});
+
+test('canonical document excludes bot comments from dedupe text', () => {
+  const document = buildCanonicalDocument({
+    title: 'Downloader stalls',
+    body: 'The transfer never finishes.',
+    labels: ['bug'],
+    comments: [
+      { body: 'same failure on macOS', authorLogin: 'alice', authorType: 'User', isBot: false },
+      { body: 'automated reminder', authorLogin: 'github-actions[bot]', authorType: 'Bot', isBot: true },
+    ],
+  });
+
+  assert.match(document.rawText, /same failure on macOS/);
+  assert.doesNotMatch(document.dedupeText, /automated reminder/);
+});

package/src/documents/normalize.ts

@@ -0,0 +1,52 @@
+import crypto from 'node:crypto';
+
+export type NormalizedComment = {
+  body: string;
+  authorLogin: string | null;
+  authorType: string | null;
+  isBot: boolean;
+};
+
+export type NormalizedThread = {
+  title: string;
+  body: string | null;
+  labels: string[];
+  comments: NormalizedComment[];
+};
+
+function normalizeWhitespace(value: string): string {
+  return value.replace(/\r/g, '\n').replace(/\s+/g, ' ').trim();
+}
+
+export function isBotLikeAuthor(input: { authorType?: string | null; authorLogin?: string | null; isBot?: boolean }): boolean {
+  if (input.isBot) return true;
+  if ((input.authorType ?? '').toLowerCase() === 'bot') return true;
+  const login = (input.authorLogin ?? '').toLowerCase();
+  return login.endsWith('[bot]') || login.includes('renovate') || login.includes('dependabot');
+}
+
+export function buildCanonicalDocument(thread: NormalizedThread): { rawText: string; dedupeText: string; contentHash: string } {
+  const labels = thread.labels.length > 0 ? `labels: ${thread.labels.join(', ')}` : '';
+  const humanComments = thread.comments
+    .filter((comment) => !isBotLikeAuthor(comment))
+    .map((comment) => {
+      const author = comment.authorLogin ? `@${comment.authorLogin}` : 'unknown';
+      return `${author}: ${normalizeWhitespace(comment.body)}`;
+    })
+    .filter(Boolean);
+
+  const title = normalizeWhitespace(thread.title);
+  const body = normalizeWhitespace(thread.body ?? '');
+  const rawParts = [title, body, labels, ...humanComments].filter(Boolean);
+  const dedupeParts = [
+    `title: ${title}`,
+    body ? `body: ${body}` : '',
+    labels ? labels : '',
+    humanComments.length > 0 ? `discussion: ${humanComments.join('\n')}` : '',
+  ].filter(Boolean);
+
+  const rawText = rawParts.join('\n\n');
+  const dedupeText = dedupeParts.join('\n\n');
+  const contentHash = crypto.createHash('sha256').update(`${rawText}\n---\n${dedupeText}`).digest('hex');
+  return { rawText, dedupeText, contentHash };
+}

package/src/github/client.ts

@@ -0,0 +1,241 @@
+import { retry } from '@octokit/plugin-retry';
+import { throttling } from '@octokit/plugin-throttling';
+import { Octokit } from 'octokit';
+
+export type GitHubClient = {
+  checkAuth: (reporter?: GitHubReporter) => Promise<void>;
+  getRepo: (owner: string, repo: string, reporter?: GitHubReporter) => Promise<Record<string, unknown>>;
+  listRepositoryIssues: (
+    owner: string,
+    repo: string,
+    since?: string,
+    limit?: number,
+    reporter?: GitHubReporter,
+  ) => Promise<Array<Record<string, unknown>>>;
+  getIssue: (owner: string, repo: string, number: number, reporter?: GitHubReporter) => Promise<Record<string, unknown>>;
+  getPull: (owner: string, repo: string, number: number, reporter?: GitHubReporter) => Promise<Record<string, unknown>>;
+  listIssueComments: (owner: string, repo: string, number: number, reporter?: GitHubReporter) => Promise<Array<Record<string, unknown>>>;
+  listPullReviews: (owner: string, repo: string, number: number, reporter?: GitHubReporter) => Promise<Array<Record<string, unknown>>>;
+  listPullReviewComments: (
+    owner: string,
+    repo: string,
+    number: number,
+    reporter?: GitHubReporter,
+  ) => Promise<Array<Record<string, unknown>>>;
+};
+
+export type GitHubReporter = (message: string) => void;
+
+export class GitHubRequestError extends Error {
+  readonly status?: number;
+
+  constructor(message: string, status?: number) {
+    super(message);
+    this.name = 'GitHubRequestError';
+    this.status = status;
+  }
+}
+
+type RequestOptions = {
+  token: string;
+  userAgent?: string;
+  timeoutMs?: number;
+  pageDelayMs?: number;
+};
+
+type OctokitPage<T> = {
+  data: T[];
+};
+
+function delay(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function formatDuration(ms: number): string {
+  if (ms < 1000) return `${ms}ms`;
+  const seconds = Math.ceil(ms / 1000);
+  if (seconds < 60) return `${seconds}s`;
+  const minutes = Math.floor(seconds / 60);
+  const remainingSeconds = seconds % 60;
+  if (minutes < 60) return remainingSeconds === 0 ? `${minutes}m` : `${minutes}m ${remainingSeconds}s`;
+  const hours = Math.floor(minutes / 60);
+  const remainingMinutes = minutes % 60;
+  return remainingMinutes === 0 ? `${hours}h` : `${hours}h ${remainingMinutes}m`;
+}
+
+function formatResetTime(resetSeconds: string | null | undefined): string | null {
+  if (!resetSeconds) return null;
+  const value = Number(resetSeconds);
+  if (!Number.isFinite(value) || value <= 0) return null;
+  return new Date(value * 1000).toISOString();
+}
+
+export function makeGitHubClient(options: RequestOptions): GitHubClient {
+  const userAgent = options.userAgent ?? 'ghcrawl';
+  const timeoutMs = options.timeoutMs ?? 30_000;
+  const pageDelayMs = options.pageDelayMs ?? 5000;
+  const BaseOctokit = Octokit.plugin(retry, throttling);
+
+  function createOctokit(reporter?: GitHubReporter) {
+    return new BaseOctokit({
+      auth: options.token,
+      request: {
+        timeout: timeoutMs,
+      },
+      userAgent,
+      retry: {
+        doNotRetry: [400, 401, 403, 404, 422],
+        retries: 4,
+      },
+      throttle: {
+        fallbackSecondaryRateRetryAfter: Math.ceil(pageDelayMs / 1000),
+        onRateLimit: (retryAfter, requestOptions) => {
+          const responseHeaders = (requestOptions.response as { headers?: Record<string, string> } | undefined)?.headers;
+          const resetAt = formatResetTime(responseHeaders?.['x-ratelimit-reset']);
+          const remaining = responseHeaders?.['x-ratelimit-remaining'];
+          const method = requestOptions.method ?? 'GET';
+          const url = requestOptions.url ?? 'unknown';
+          reporter?.(
+            `[github] backoff rate-limited wait=${formatDuration(retryAfter * 1000)}${remaining ? ` remaining=${remaining}` : ''}${resetAt ? ` reset_at=${resetAt}` : ''} method=${method} url=${url}`,
+          );
+          return true;
+        },
+        onSecondaryRateLimit: (retryAfter, requestOptions) => {
+          const method = requestOptions.method ?? 'GET';
+          const url = requestOptions.url ?? 'unknown';
+          reporter?.(
+            `[github] backoff secondary-rate-limit wait=${formatDuration(retryAfter * 1000)} method=${method} url=${url}`,
+          );
+          return true;
+        },
+      },
+    });
+  }
+
+  async function request<T>(label: string, reporter: GitHubReporter | undefined, fn: (octokit: InstanceType<typeof BaseOctokit>) => Promise<T>): Promise<T> {
+    reporter?.(`[github] request ${label}`);
+    const octokit = createOctokit(reporter);
+    try {
+      return await fn(octokit);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      const status = typeof (error as { status?: unknown })?.status === 'number' ? Number((error as { status?: unknown }).status) : undefined;
+      throw new GitHubRequestError(`GitHub request failed for ${label}: ${message}`, status);
+    }
+  }
+
+  async function paginate<T>(
+    label: string,
+    limit: number | undefined,
+    reporter: GitHubReporter | undefined,
+    iteratorFactory: (octokit: InstanceType<typeof BaseOctokit>) => AsyncIterable<OctokitPage<T>>,
+  ): Promise<T[]> {
+    reporter?.(`[github] request ${label}`);
+    const octokit = createOctokit(reporter);
+    const out: T[] = [];
+
+    try {
+      let pageIndex = 0;
+      for await (const page of iteratorFactory(octokit)) {
+        pageIndex += 1;
+        const remaining = typeof limit === 'number' ? Math.max(limit - out.length, 0) : page.data.length;
+        out.push(...page.data.slice(0, remaining));
+        reporter?.(`[github] page ${pageIndex} fetched count=${page.data.length} accumulated=${out.length}`);
+        if (typeof limit === 'number' && out.length >= limit) {
+          break;
+        }
+        await delay(pageDelayMs);
+      }
+      return out;
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      const status = typeof (error as { status?: unknown })?.status === 'number' ? Number((error as { status?: unknown }).status) : undefined;
+      throw new GitHubRequestError(`GitHub pagination failed for ${label}: ${message}`, status);
+    }
+  }
+
+  return {
+    async checkAuth(reporter) {
+      await request('GET /rate_limit', reporter, async (octokit) => {
+        await octokit.request('GET /rate_limit');
+      });
+    },
+    async getRepo(owner, repo, reporter) {
+      return request(`GET /repos/${owner}/${repo}`, reporter, async (octokit) => {
+        const response = await octokit.rest.repos.get({ owner, repo });
+        return response.data as Record<string, unknown>;
+      });
+    },
+    async listRepositoryIssues(owner, repo, since, limit, reporter) {
+      return paginate(
+        `GET /repos/${owner}/${repo}/issues state=open per_page=100`,
+        limit,
+        reporter,
+        (octokit) =>
+          octokit.paginate.iterator(octokit.rest.issues.listForRepo, {
+            owner,
+            repo,
+            state: 'open',
+            sort: 'updated',
+            direction: 'desc',
+            per_page: 100,
+            since,
+          }) as AsyncIterable<OctokitPage<Record<string, unknown>>>,
+      );
+    },
+    async getIssue(owner, repo, number, reporter) {
+      return request(`GET /repos/${owner}/${repo}/issues/${number}`, reporter, async (octokit) => {
+        const response = await octokit.rest.issues.get({ owner, repo, issue_number: number });
+        return response.data as Record<string, unknown>;
+      });
+    },
+    async getPull(owner, repo, number, reporter) {
+      return request(`GET /repos/${owner}/${repo}/pulls/${number}`, reporter, async (octokit) => {
+        const response = await octokit.rest.pulls.get({ owner, repo, pull_number: number });
+        return response.data as Record<string, unknown>;
+      });
+    },
+    async listIssueComments(owner, repo, number, reporter) {
+      return paginate(
+        `GET /repos/${owner}/${repo}/issues/${number}/comments per_page=100`,
+        undefined,
+        reporter,
+        (octokit) =>
+          octokit.paginate.iterator(octokit.rest.issues.listComments, {
+            owner,
+            repo,
+            issue_number: number,
+            per_page: 100,
+          }) as AsyncIterable<OctokitPage<Record<string, unknown>>>,
+      );
+    },
+    async listPullReviews(owner, repo, number, reporter) {
+      return paginate(
+        `GET /repos/${owner}/${repo}/pulls/${number}/reviews per_page=100`,
+        undefined,
+        reporter,
+        (octokit) =>
+          octokit.paginate.iterator(octokit.rest.pulls.listReviews, {
+            owner,
+            repo,
+            pull_number: number,
+            per_page: 100,
+          }) as AsyncIterable<OctokitPage<Record<string, unknown>>>,
+      );
+    },
+    async listPullReviewComments(owner, repo, number, reporter) {
+      return paginate(
+        `GET /repos/${owner}/${repo}/pulls/${number}/comments per_page=100`,
+        undefined,
+        reporter,
+        (octokit) =>
+          octokit.paginate.iterator(octokit.rest.pulls.listReviewComments, {
+            owner,
+            repo,
+            pull_number: number,
+            per_page: 100,
+          }) as AsyncIterable<OctokitPage<Record<string, unknown>>>,
+      );
+    },
+  };
+}

package/src/index.ts

@@ -0,0 +1,6 @@
+export * from './api/server.js';
+export * from './config.js';
+export * from './documents/normalize.js';
+export * from './search/exact.js';
+export * from './cluster/build.js';
+export * from './service.js';