@ghcrawl/api-core 0.1.0 → 0.2.0

package/src/github/client.ts

@@ -1,241 +0,0 @@
-import { retry } from '@octokit/plugin-retry';
-import { throttling } from '@octokit/plugin-throttling';
-import { Octokit } from 'octokit';
-
-export type GitHubClient = {
-  checkAuth: (reporter?: GitHubReporter) => Promise<void>;
-  getRepo: (owner: string, repo: string, reporter?: GitHubReporter) => Promise<Record<string, unknown>>;
-  listRepositoryIssues: (
-    owner: string,
-    repo: string,
-    since?: string,
-    limit?: number,
-    reporter?: GitHubReporter,
-  ) => Promise<Array<Record<string, unknown>>>;
-  getIssue: (owner: string, repo: string, number: number, reporter?: GitHubReporter) => Promise<Record<string, unknown>>;
-  getPull: (owner: string, repo: string, number: number, reporter?: GitHubReporter) => Promise<Record<string, unknown>>;
-  listIssueComments: (owner: string, repo: string, number: number, reporter?: GitHubReporter) => Promise<Array<Record<string, unknown>>>;
-  listPullReviews: (owner: string, repo: string, number: number, reporter?: GitHubReporter) => Promise<Array<Record<string, unknown>>>;
-  listPullReviewComments: (
-    owner: string,
-    repo: string,
-    number: number,
-    reporter?: GitHubReporter,
-  ) => Promise<Array<Record<string, unknown>>>;
-};
-
-export type GitHubReporter = (message: string) => void;
-
-export class GitHubRequestError extends Error {
-  readonly status?: number;
-
-  constructor(message: string, status?: number) {
-    super(message);
-    this.name = 'GitHubRequestError';
-    this.status = status;
-  }
-}
-
-type RequestOptions = {
-  token: string;
-  userAgent?: string;
-  timeoutMs?: number;
-  pageDelayMs?: number;
-};
-
-type OctokitPage<T> = {
-  data: T[];
-};
-
-function delay(ms: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
-function formatDuration(ms: number): string {
-  if (ms < 1000) return `${ms}ms`;
-  const seconds = Math.ceil(ms / 1000);
-  if (seconds < 60) return `${seconds}s`;
-  const minutes = Math.floor(seconds / 60);
-  const remainingSeconds = seconds % 60;
-  if (minutes < 60) return remainingSeconds === 0 ? `${minutes}m` : `${minutes}m ${remainingSeconds}s`;
-  const hours = Math.floor(minutes / 60);
-  const remainingMinutes = minutes % 60;
-  return remainingMinutes === 0 ? `${hours}h` : `${hours}h ${remainingMinutes}m`;
-}
-
-function formatResetTime(resetSeconds: string | null | undefined): string | null {
-  if (!resetSeconds) return null;
-  const value = Number(resetSeconds);
-  if (!Number.isFinite(value) || value <= 0) return null;
-  return new Date(value * 1000).toISOString();
-}
-
-export function makeGitHubClient(options: RequestOptions): GitHubClient {
-  const userAgent = options.userAgent ?? 'ghcrawl';
-  const timeoutMs = options.timeoutMs ?? 30_000;
-  const pageDelayMs = options.pageDelayMs ?? 5000;
-  const BaseOctokit = Octokit.plugin(retry, throttling);
-
-  function createOctokit(reporter?: GitHubReporter) {
-    return new BaseOctokit({
-      auth: options.token,
-      request: {
-        timeout: timeoutMs,
-      },
-      userAgent,
-      retry: {
-        doNotRetry: [400, 401, 403, 404, 422],
-        retries: 4,
-      },
-      throttle: {
-        fallbackSecondaryRateRetryAfter: Math.ceil(pageDelayMs / 1000),
-        onRateLimit: (retryAfter, requestOptions) => {
-          const responseHeaders = (requestOptions.response as { headers?: Record<string, string> } | undefined)?.headers;
-          const resetAt = formatResetTime(responseHeaders?.['x-ratelimit-reset']);
-          const remaining = responseHeaders?.['x-ratelimit-remaining'];
-          const method = requestOptions.method ?? 'GET';
-          const url = requestOptions.url ?? 'unknown';
-          reporter?.(
-            `[github] backoff rate-limited wait=${formatDuration(retryAfter * 1000)}${remaining ? ` remaining=${remaining}` : ''}${resetAt ? ` reset_at=${resetAt}` : ''} method=${method} url=${url}`,
-          );
-          return true;
-        },
-        onSecondaryRateLimit: (retryAfter, requestOptions) => {
-          const method = requestOptions.method ?? 'GET';
-          const url = requestOptions.url ?? 'unknown';
-          reporter?.(
-            `[github] backoff secondary-rate-limit wait=${formatDuration(retryAfter * 1000)} method=${method} url=${url}`,
-          );
-          return true;
-        },
-      },
-    });
-  }
-
-  async function request<T>(label: string, reporter: GitHubReporter | undefined, fn: (octokit: InstanceType<typeof BaseOctokit>) => Promise<T>): Promise<T> {
-    reporter?.(`[github] request ${label}`);
-    const octokit = createOctokit(reporter);
-    try {
-      return await fn(octokit);
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      const status = typeof (error as { status?: unknown })?.status === 'number' ? Number((error as { status?: unknown }).status) : undefined;
-      throw new GitHubRequestError(`GitHub request failed for ${label}: ${message}`, status);
-    }
-  }
-
-  async function paginate<T>(
-    label: string,
-    limit: number | undefined,
-    reporter: GitHubReporter | undefined,
-    iteratorFactory: (octokit: InstanceType<typeof BaseOctokit>) => AsyncIterable<OctokitPage<T>>,
-  ): Promise<T[]> {
-    reporter?.(`[github] request ${label}`);
-    const octokit = createOctokit(reporter);
-    const out: T[] = [];
-
-    try {
-      let pageIndex = 0;
-      for await (const page of iteratorFactory(octokit)) {
-        pageIndex += 1;
-        const remaining = typeof limit === 'number' ? Math.max(limit - out.length, 0) : page.data.length;
-        out.push(...page.data.slice(0, remaining));
-        reporter?.(`[github] page ${pageIndex} fetched count=${page.data.length} accumulated=${out.length}`);
-        if (typeof limit === 'number' && out.length >= limit) {
-          break;
-        }
-        await delay(pageDelayMs);
-      }
-      return out;
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      const status = typeof (error as { status?: unknown })?.status === 'number' ? Number((error as { status?: unknown }).status) : undefined;
-      throw new GitHubRequestError(`GitHub pagination failed for ${label}: ${message}`, status);
-    }
-  }
-
-  return {
-    async checkAuth(reporter) {
-      await request('GET /rate_limit', reporter, async (octokit) => {
-        await octokit.request('GET /rate_limit');
-      });
-    },
-    async getRepo(owner, repo, reporter) {
-      return request(`GET /repos/${owner}/${repo}`, reporter, async (octokit) => {
-        const response = await octokit.rest.repos.get({ owner, repo });
-        return response.data as Record<string, unknown>;
-      });
-    },
-    async listRepositoryIssues(owner, repo, since, limit, reporter) {
-      return paginate(
-        `GET /repos/${owner}/${repo}/issues state=open per_page=100`,
-        limit,
-        reporter,
-        (octokit) =>
-          octokit.paginate.iterator(octokit.rest.issues.listForRepo, {
-            owner,
-            repo,
-            state: 'open',
-            sort: 'updated',
-            direction: 'desc',
-            per_page: 100,
-            since,
-          }) as AsyncIterable<OctokitPage<Record<string, unknown>>>,
-      );
-    },
-    async getIssue(owner, repo, number, reporter) {
-      return request(`GET /repos/${owner}/${repo}/issues/${number}`, reporter, async (octokit) => {
-        const response = await octokit.rest.issues.get({ owner, repo, issue_number: number });
-        return response.data as Record<string, unknown>;
-      });
-    },
-    async getPull(owner, repo, number, reporter) {
-      return request(`GET /repos/${owner}/${repo}/pulls/${number}`, reporter, async (octokit) => {
-        const response = await octokit.rest.pulls.get({ owner, repo, pull_number: number });
-        return response.data as Record<string, unknown>;
-      });
-    },
-    async listIssueComments(owner, repo, number, reporter) {
-      return paginate(
-        `GET /repos/${owner}/${repo}/issues/${number}/comments per_page=100`,
-        undefined,
-        reporter,
-        (octokit) =>
-          octokit.paginate.iterator(octokit.rest.issues.listComments, {
-            owner,
-            repo,
-            issue_number: number,
-            per_page: 100,
-          }) as AsyncIterable<OctokitPage<Record<string, unknown>>>,
-      );
-    },
-    async listPullReviews(owner, repo, number, reporter) {
-      return paginate(
-        `GET /repos/${owner}/${repo}/pulls/${number}/reviews per_page=100`,
-        undefined,
-        reporter,
-        (octokit) =>
-          octokit.paginate.iterator(octokit.rest.pulls.listReviews, {
-            owner,
-            repo,
-            pull_number: number,
-            per_page: 100,
-          }) as AsyncIterable<OctokitPage<Record<string, unknown>>>,
-      );
-    },
-    async listPullReviewComments(owner, repo, number, reporter) {
-      return paginate(
-        `GET /repos/${owner}/${repo}/pulls/${number}/comments per_page=100`,
-        undefined,
-        reporter,
-        (octokit) =>
-          octokit.paginate.iterator(octokit.rest.pulls.listReviewComments, {
-            owner,
-            repo,
-            pull_number: number,
-            per_page: 100,
-          }) as AsyncIterable<OctokitPage<Record<string, unknown>>>,
-      );
-    },
-  };
-}

package/src/openai/provider.ts

@@ -1,141 +0,0 @@
-import OpenAI from 'openai';
-import { APIConnectionError, APIConnectionTimeoutError, APIError, RateLimitError } from 'openai/error';
-import { zodTextFormat } from 'openai/helpers/zod';
-import { z } from 'zod';
-
-export type SummaryResult = {
-  problemSummary: string;
-  solutionSummary: string;
-  maintainerSignalSummary: string;
-  dedupeSummary: string;
-};
-
-export type SummaryUsage = {
-  inputTokens: number;
-  outputTokens: number;
-  totalTokens: number;
-  cachedInputTokens: number;
-  reasoningTokens: number;
-};
-
-export type AiProvider = {
-  checkAuth: () => Promise<void>;
-  summarizeThread: (params: { model: string; text: string }) => Promise<{ summary: SummaryResult; usage?: SummaryUsage }>;
-  embedTexts: (params: { model: string; texts: string[] }) => Promise<number[][]>;
-};
-
-const summarySchema = z.object({
-  problem_summary: z.string(),
-  solution_summary: z.string(),
-  maintainer_signal_summary: z.string(),
-  dedupe_summary: z.string(),
-});
-
-export class OpenAiProvider implements AiProvider {
-  private readonly client: OpenAI;
-
-  constructor(apiKey: string) {
-    this.client = new OpenAI({ apiKey });
-  }
-
-  async checkAuth(): Promise<void> {
-    await this.client.models.list();
-  }
-
-  async summarizeThread(params: { model: string; text: string }): Promise<{ summary: SummaryResult; usage?: SummaryUsage }> {
-    const format = zodTextFormat(summarySchema, 'ghcrawl_thread_summary');
-    let lastError: Error | null = null;
-
-    for (const [attemptIndex, maxOutputTokens] of [500, 900, 1400].entries()) {
-      try {
-        const response = await this.client.responses.create({
-          model: params.model,
-          input: [
-            {
-              role: 'system',
-              content: [
-                {
-                  type: 'input_text',
-                  text:
-                    'Summarize this GitHub issue or pull request thread. Return concise JSON only with keys problem_summary, solution_summary, maintainer_signal_summary, dedupe_summary. Each field should be plain text, no markdown, and usually 1-3 sentences.',
-                },
-              ],
-            },
-            {
-              role: 'user',
-              content: [{ type: 'input_text', text: params.text }],
-            },
-          ],
-          text: {
-            format,
-            verbosity: 'low',
-          },
-          max_output_tokens: maxOutputTokens,
-        });
-
-        const raw = response.output_text ?? '';
-        const parsed = summarySchema.parse(JSON.parse(raw));
-
-        return {
-          summary: {
-            problemSummary: parsed.problem_summary,
-            solutionSummary: parsed.solution_summary,
-            maintainerSignalSummary: parsed.maintainer_signal_summary,
-            dedupeSummary: parsed.dedupe_summary,
-          },
-          usage: response.usage
-            ? {
-                inputTokens: response.usage.input_tokens,
-                outputTokens: response.usage.output_tokens,
-                totalTokens: response.usage.total_tokens,
-                cachedInputTokens: response.usage.input_tokens_details?.cached_tokens ?? 0,
-                reasoningTokens: response.usage.output_tokens_details?.reasoning_tokens ?? 0,
-              }
-            : undefined,
-        };
-      } catch (error) {
-        lastError = error instanceof Error ? error : new Error(String(error));
-        if (attemptIndex === 2) {
-          break;
-        }
-      }
-    }
-
-    throw new Error(`OpenAI summarization failed after 3 attempts: ${lastError?.message ?? 'unknown error'}`);
-  }
-
-  async embedTexts(params: { model: string; texts: string[] }): Promise<number[][]> {
-    if (params.texts.length === 0) {
-      return [];
-    }
-
-    let lastError: Error | null = null;
-    for (const attempt of [1, 2, 3, 4, 5]) {
-      try {
-        const response = await this.client.embeddings.create({
-          model: params.model,
-          input: params.texts,
-        });
-
-        return response.data.map((item) => item.embedding);
-      } catch (error) {
-        const shouldRetry =
-          error instanceof RateLimitError ||
-          error instanceof APIConnectionError ||
-          error instanceof APIConnectionTimeoutError ||
-          (error instanceof APIError && typeof error.status === 'number' && error.status >= 500);
-        lastError = error instanceof Error ? error : new Error(String(error));
-        if (!shouldRetry || attempt === 5) {
-          break;
-        }
-        await sleep(1000 * 2 ** (attempt - 1));
-      }
-    }
-
-    throw new Error(`OpenAI embeddings failed after 5 attempts: ${lastError?.message ?? 'unknown error'}`);
-  }
-}
-
-function sleep(ms: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}

package/src/search/exact.test.ts

@@ -1,22 +0,0 @@
-import test from 'node:test';
-import assert from 'node:assert/strict';
-
-import { cosineSimilarity, rankNearestNeighbors } from './exact.js';
-
-test('cosine similarity is 1 for identical embeddings', () => {
-  assert.equal(cosineSimilarity([1, 0], [1, 0]), 1);
-});
-
-test('nearest neighbors sorts by similarity descending', () => {
-  const ranked = rankNearestNeighbors(
-    [
-      { id: 1, embedding: [1, 0] },
-      { id: 2, embedding: [0.9, 0.1] },
-      { id: 3, embedding: [0, 1] },
-    ],
-    { targetEmbedding: [1, 0], limit: 2, skipId: 1 },
-  );
-
-  assert.equal(ranked[0]?.item.id, 2);
-  assert.equal(ranked[1]?.item.id, 3);
-});

package/src/search/exact.ts

@@ -1,28 +0,0 @@
-export function cosineSimilarity(left: number[], right: number[]): number {
-  if (left.length !== right.length) {
-    throw new Error('Embedding dimensions do not match');
-  }
-  let dot = 0;
-  let leftNorm = 0;
-  let rightNorm = 0;
-  for (let index = 0; index < left.length; index += 1) {
-    dot += left[index] * right[index];
-    leftNorm += left[index] * left[index];
-    rightNorm += right[index] * right[index];
-  }
-  if (leftNorm === 0 || rightNorm === 0) return 0;
-  return dot / (Math.sqrt(leftNorm) * Math.sqrt(rightNorm));
-}
-
-export function rankNearestNeighbors<T extends { id: number; embedding: number[] }>(
-  items: T[],
-  params: { targetEmbedding: number[]; limit: number; minScore?: number; skipId?: number },
-): Array<{ item: T; score: number }> {
-  const minScore = params.minScore ?? -1;
-  return items
-    .filter((item) => item.id !== params.skipId)
-    .map((item) => ({ item, score: cosineSimilarity(params.targetEmbedding, item.embedding) }))
-    .filter((entry) => entry.score >= minScore)
-    .sort((left, right) => right.score - left.score)
-    .slice(0, params.limit);
-}