@pratik7368patil/anchor-core 0.1.12 → 0.1.13

package/dist/index.d.ts

@@ -244,6 +244,14 @@ type FetchPullRequestsProgress = {
     total: number;
     prNumber: number;
     detailConcurrency: number;
+} | {
+    stage: "github_rate_limited";
+    repo: string;
+    waitSeconds: number;
+    retryAt: string;
+    reason: string;
+    request: string;
+    attempt: number;
 };
 type IndexPullRequestsProgress = {
     stage: "indexing_pull_request";
@@ -738,6 +746,45 @@ declare const DEMO_CODE_FILES: Record<string, string>;
 
 declare function createGitHubClient(token: string): Octokit;
 
+type GitHubRateLimitProgress = {
+    waitSeconds: number;
+    retryAt: string;
+    reason: string;
+    request: string;
+    attempt: number;
+};
+type GitHubRateLimitController = {
+    onRateLimit?: (progress: GitHubRateLimitProgress) => void;
+    sleep?: (milliseconds: number) => Promise<void>;
+    now?: () => number;
+    blockedUntilMs?: number;
+};
+type GitHubRateLimitErrorLike = {
+    status?: number;
+    message?: string;
+    response?: {
+        headers?: Record<string, string | number | undefined>;
+    };
+};
+type GitHubResponse<T> = {
+    data: T;
+    headers: Record<string, string | number | undefined>;
+};
+declare function isGitHubRateLimitError(error: unknown): error is GitHubRateLimitErrorLike;
+declare function getGitHubRateLimitDelayMs(error: GitHubRateLimitErrorLike, attempt: number, now?: number): {
+    delayMs: number;
+    reason: string;
+};
+declare function requestWithGitHubRateLimit<T>(request: () => Promise<T>, options: {
+    controller: GitHubRateLimitController;
+    requestName: string;
+    maxRetries?: number;
+}): Promise<T>;
+declare function paginateWithGitHubRateLimit<T>(requestPage: (page: number) => Promise<GitHubResponse<T[]>>, options: {
+    controller: GitHubRateLimitController;
+    requestName: string;
+}): Promise<T[]>;
+
 type FetchPullRequestsOptions = {
     token: string;
     repo: string;
@@ -751,7 +798,7 @@ declare function resolvePullRequestFetchLimit(options: Pick<FetchPullRequestsOpt
 declare function resolvePullRequestDetailConcurrency(options: Pick<FetchPullRequestsOptions, "detailConcurrency">): number;
 declare function fetchMergedPullRequests(options: FetchPullRequestsOptions): Promise<PullRequestRecord[]>;
 
-declare function fetchPullRequestDetails(octokit: Octokit, repoFullName: string, pullNumber: number): Promise<PullRequestRecord>;
+declare function fetchPullRequestDetails(octokit: Octokit, repoFullName: string, pullNumber: number, controller?: GitHubRateLimitController): Promise<PullRequestRecord>;
 
 type DoctorOptions = {
     cwd: string;
@@ -766,4 +813,4 @@ declare function getAnchorIndexHealth(cwd: string): AnchorIndexHealth & {
     indexStatus: IndexStatus;
 };
 
-export { ANCHOR_CURSOR_RULE, type AnchorContextInput, type AnchorDatabase, type AnchorExplainFileInput, type AnchorIndexHealth, type AnchorReviewDiffInput, type ArchitectureArea, type ArchitectureCheckInput, type ArchitectureComponent, type ArchitectureContextInput, type ArchitectureIndexData, type ArchitecturePattern, type ArchitectureQueryInput, type ChunkableCodeFile, type CodeChunk, type CodeFileDiscoveryResult, type CodeFileRecord, type CodeImport, type CodeIndexProgress, type CodeIndexSummary, type ConfidenceLevel, type CoverageGrade, type CoverageInput, type CoverageReport, type CurrentCodeSnapshot, type CursorMcpConfig, DEFAULT_MAX_CODE_FILE_BYTES, DEMO_CODE_FILES, DEMO_PULL_REQUESTS, DEMO_REPO, type DiscoveredCodeFile, type DoctorCheck, type DoctorOptions, type DoctorReport, type EvidenceRef, type FetchPullRequestsOptions, type FetchPullRequestsProgress, type FormattedResult, type FreshnessResult, type FreshnessStatus, type GitHubRepo, type GitHubTokenResolution, type GitHubTokenResolverOptions, type GitHubTokenSource, type IndexPullRequestsProgress, type IndexRunRecord, type IndexStatus, type IndexSummary, type LocalEmbeddingProvider, type PullRequestComment, type PullRequestCommit, type PullRequestFile, type PullRequestPerson, type PullRequestRecord, type RankedArchitecturePattern, type RankedCodeChunk, type RankedRegressionEvent, type RankedTeamRule, type RankedTestFile, type RankedWisdomUnit, type RegressionEvent, type RulesAddInput, type RulesAddResult, type RulesEvidenceCheckResult, type RulesInitResult, type RulesSuggestOptions, SCHEMA_SQL, type SearchHistoryInput, type SemanticStatus, type SourceType, type SuggestedPrompt, TEAM_RULES_FILE, type TeamRule, type TeamRuleSuggestion, type TeamRulesValidationResult, type TestFileRecord, type TestLink, type WisdomCategory, type WisdomUnit, addTeamRule, anchorMcpEntry, architectureFilesFromDiff, buildAnchorContextResult, buildArchitectureIndex, buildFtsQuery, buildQueryTerms, calculateCoverage, canonicalizeText, categorizeWisdom, checkArchitecture, checkSchema, checkTeamRuleEvidence, chunkCodeFile, chunkHistoricalText, claimKeyFor, clampMaxResults, classifyArchitectureArea, clipSentence, confidenceAtLeast, confidenceLevelFor, confidenceRank, confidenceReasonsFor, countValidTeamRules, createGitHubClient, defaultDatabasePath, detectGitHubRepo, detectGitRoot, discoverCodeFiles, emptyCodeIndexSummary, ensureAnchorGitExclude, ensureCursorConfig, ensureCursorRule, ensureRepository, ensureTeamRulesFile, evaluateFreshness, evaluateIndexHealth, evidenceForWisdom, explainFile, extractCodeImports, extractCodeSymbols, extractRegressionEvents, extractSymbols, extractWisdomUnits, fetchMergedPullRequests, fetchPullRequestDetails, filesFromDiff, formatAnchorContext, formatIndexStatus, formatSearchHistory, getAnchorIndexHealth, getArchitectureContext, getIndexStatus, getLastSyncTime, getSemanticStatus, getSuggestedPromptTexts, getSuggestedPrompts, getWisdomCategoryCounts, githubAuthFixMessage, hasHighSignalLanguage, indexCodebase, indexPullRequests, inferTestAwareness, initializeSchema, isHardExcludedCodePath, isTestFilePath, loadCurrentCodeSnapshot, loadTeamRulesFile, mergeAnchorMcpConfig, normalizePullRequest, openAnchorDatabase, parseGitHubRemote, rankArchitecturePatterns, rankCodeChunks, rankRegressionEvents, rankRelevantTests, rankTeamRules, rankWisdomUnits, recordIndexRun, redactSecrets, redactedHistoricalText, replaceCodeIndex, resolveGitHubToken, resolvePullRequestDetailConcurrency, resolvePullRequestFetchLimit, reviewDiff, runDoctor, sanitizeHistoricalText, shouldSyncSince, sourceTypeLabel, stripPromptInjection, suggestTeamRules, tokenizeSearchText, truncateText, uniqueStrings, updateSyncState, upsertPullRequest, validateTeamRulesFile };
+export { ANCHOR_CURSOR_RULE, type AnchorContextInput, type AnchorDatabase, type AnchorExplainFileInput, type AnchorIndexHealth, type AnchorReviewDiffInput, type ArchitectureArea, type ArchitectureCheckInput, type ArchitectureComponent, type ArchitectureContextInput, type ArchitectureIndexData, type ArchitecturePattern, type ArchitectureQueryInput, type ChunkableCodeFile, type CodeChunk, type CodeFileDiscoveryResult, type CodeFileRecord, type CodeImport, type CodeIndexProgress, type CodeIndexSummary, type ConfidenceLevel, type CoverageGrade, type CoverageInput, type CoverageReport, type CurrentCodeSnapshot, type CursorMcpConfig, DEFAULT_MAX_CODE_FILE_BYTES, DEMO_CODE_FILES, DEMO_PULL_REQUESTS, DEMO_REPO, type DiscoveredCodeFile, type DoctorCheck, type DoctorOptions, type DoctorReport, type EvidenceRef, type FetchPullRequestsOptions, type FetchPullRequestsProgress, type FormattedResult, type FreshnessResult, type FreshnessStatus, type GitHubRateLimitController, type GitHubRateLimitErrorLike, type GitHubRateLimitProgress, type GitHubRepo, type GitHubTokenResolution, type GitHubTokenResolverOptions, type GitHubTokenSource, type IndexPullRequestsProgress, type IndexRunRecord, type IndexStatus, type IndexSummary, type LocalEmbeddingProvider, type PullRequestComment, type PullRequestCommit, type PullRequestFile, type PullRequestPerson, type PullRequestRecord, type RankedArchitecturePattern, type RankedCodeChunk, type RankedRegressionEvent, type RankedTeamRule, type RankedTestFile, type RankedWisdomUnit, type RegressionEvent, type RulesAddInput, type RulesAddResult, type RulesEvidenceCheckResult, type RulesInitResult, type RulesSuggestOptions, SCHEMA_SQL, type SearchHistoryInput, type SemanticStatus, type SourceType, type SuggestedPrompt, TEAM_RULES_FILE, type TeamRule, type TeamRuleSuggestion, type TeamRulesValidationResult, type TestFileRecord, type TestLink, type WisdomCategory, type WisdomUnit, addTeamRule, anchorMcpEntry, architectureFilesFromDiff, buildAnchorContextResult, buildArchitectureIndex, buildFtsQuery, buildQueryTerms, calculateCoverage, canonicalizeText, categorizeWisdom, checkArchitecture, checkSchema, checkTeamRuleEvidence, chunkCodeFile, chunkHistoricalText, claimKeyFor, clampMaxResults, classifyArchitectureArea, clipSentence, confidenceAtLeast, confidenceLevelFor, confidenceRank, confidenceReasonsFor, countValidTeamRules, createGitHubClient, defaultDatabasePath, detectGitHubRepo, detectGitRoot, discoverCodeFiles, emptyCodeIndexSummary, ensureAnchorGitExclude, ensureCursorConfig, ensureCursorRule, ensureRepository, ensureTeamRulesFile, evaluateFreshness, evaluateIndexHealth, evidenceForWisdom, explainFile, extractCodeImports, extractCodeSymbols, extractRegressionEvents, extractSymbols, extractWisdomUnits, fetchMergedPullRequests, fetchPullRequestDetails, filesFromDiff, formatAnchorContext, formatIndexStatus, formatSearchHistory, getAnchorIndexHealth, getArchitectureContext, getGitHubRateLimitDelayMs, getIndexStatus, getLastSyncTime, getSemanticStatus, getSuggestedPromptTexts, getSuggestedPrompts, getWisdomCategoryCounts, githubAuthFixMessage, hasHighSignalLanguage, indexCodebase, indexPullRequests, inferTestAwareness, initializeSchema, isGitHubRateLimitError, isHardExcludedCodePath, isTestFilePath, loadCurrentCodeSnapshot, loadTeamRulesFile, mergeAnchorMcpConfig, normalizePullRequest, openAnchorDatabase, paginateWithGitHubRateLimit, parseGitHubRemote, rankArchitecturePatterns, rankCodeChunks, rankRegressionEvents, rankRelevantTests, rankTeamRules, rankWisdomUnits, recordIndexRun, redactSecrets, redactedHistoricalText, replaceCodeIndex, requestWithGitHubRateLimit, resolveGitHubToken, resolvePullRequestDetailConcurrency, resolvePullRequestFetchLimit, reviewDiff, runDoctor, sanitizeHistoricalText, shouldSyncSince, sourceTypeLabel, stripPromptInjection, suggestTeamRules, tokenizeSearchText, truncateText, uniqueStrings, updateSyncState, upsertPullRequest, validateTeamRulesFile };

package/dist/index.js

@@ -4554,28 +4554,168 @@ function createGitHubClient(token) {
   });
 }
 
+// src/github/rate-limit.ts
+function isGitHubRateLimitError(error) {
+  const candidate = error;
+  if (candidate.status !== 403 && candidate.status !== 429) return false;
+  const message = candidate.message?.toLowerCase() ?? "";
+  const headers = candidate.response?.headers ?? {};
+  return candidate.status === 429 || headers["retry-after"] !== void 0 || headers["x-ratelimit-remaining"] === "0" || message.includes("rate limit") || message.includes("secondary limit");
+}
+function getGitHubRateLimitDelayMs(error, attempt, now = Date.now()) {
+  const headers = error.response?.headers ?? {};
+  const retryAfter = Number(headers["retry-after"]);
+  if (Number.isFinite(retryAfter) && retryAfter > 0) {
+    return {
+      delayMs: Math.ceil(retryAfter * 1e3),
+      reason: `retry-after header requested ${Math.ceil(retryAfter)} seconds`
+    };
+  }
+  const remaining = String(headers["x-ratelimit-remaining"] ?? "");
+  const reset = Number(headers["x-ratelimit-reset"]);
+  if (remaining === "0" && Number.isFinite(reset) && reset > 0) {
+    const resetDelayMs = Math.max(0, reset * 1e3 - now);
+    return {
+      delayMs: Math.ceil(resetDelayMs + 2e3),
+      reason: `primary rate limit resets at ${new Date(reset * 1e3).toISOString()}`
+    };
+  }
+  const backoffSeconds = Math.min(900, 60 * 2 ** Math.max(0, attempt - 1));
+  return {
+    delayMs: backoffSeconds * 1e3,
+    reason: `secondary rate limit backoff for ${backoffSeconds} seconds`
+  };
+}
+async function sleep(milliseconds) {
+  await new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
+async function waitForGlobalBlock(controller) {
+  const now = controller.now?.() ?? Date.now();
+  const waitMs = Math.max(0, (controller.blockedUntilMs ?? 0) - now);
+  if (waitMs > 0) {
+    await (controller.sleep ?? sleep)(waitMs);
+  }
+}
+async function requestWithGitHubRateLimit(request, options) {
+  const maxRetries = options.maxRetries ?? 8;
+  for (let attempt = 1; ; attempt += 1) {
+    await waitForGlobalBlock(options.controller);
+    try {
+      return await request();
+    } catch (error) {
+      if (!isGitHubRateLimitError(error) || attempt > maxRetries) throw error;
+      const now = options.controller.now?.() ?? Date.now();
+      const { delayMs, reason } = getGitHubRateLimitDelayMs(error, attempt, now);
+      const retryAtMs = now + delayMs;
+      options.controller.blockedUntilMs = Math.max(
+        options.controller.blockedUntilMs ?? 0,
+        retryAtMs
+      );
+      options.controller.onRateLimit?.({
+        waitSeconds: Math.ceil(delayMs / 1e3),
+        retryAt: new Date(retryAtMs).toISOString(),
+        reason,
+        request: options.requestName,
+        attempt
+      });
+      await (options.controller.sleep ?? sleep)(delayMs);
+    }
+  }
+}
+function hasNextPage(headers) {
+  return String(headers.link ?? "").includes('rel="next"');
+}
+async function paginateWithGitHubRateLimit(requestPage, options) {
+  const results = [];
+  for (let page = 1; ; page += 1) {
+    const response = await requestWithGitHubRateLimit(() => requestPage(page), {
+      controller: options.controller,
+      requestName: `${options.requestName} page ${page}`
+    });
+    results.push(...response.data);
+    if (!hasNextPage(response.headers) && response.data.length < 100) break;
+    if (!hasNextPage(response.headers) && response.data.length === 0) break;
+    if (!hasNextPage(response.headers)) break;
+  }
+  return results;
+}
+
 // src/github/fetch-pr-details.ts
-async function fetchPullRequestDetails(octokit, repoFullName, pullNumber) {
+async function fetchPullRequestDetails(octokit, repoFullName, pullNumber, controller = {}) {
   const [owner, repo] = repoFullName.split("/");
   if (!owner || !repo) throw new Error(`Invalid repo '${repoFullName}'. Expected owner/name.`);
-  const [{ data: pull }, files, reviews, reviewComments, issueComments, commits] = await Promise.all([
-    octokit.pulls.get({ owner, repo, pull_number: pullNumber }),
-    octokit.paginate(octokit.pulls.listFiles, { owner, repo, pull_number: pullNumber, per_page: 100 }),
-    octokit.paginate(octokit.pulls.listReviews, { owner, repo, pull_number: pullNumber, per_page: 100 }),
-    octokit.paginate(octokit.pulls.listReviewComments, {
+  const { data: pull } = await requestWithGitHubRateLimit(
+    () => octokit.pulls.get({ owner, repo, pull_number: pullNumber }),
+    {
+      controller,
+      requestName: `GET /repos/${repoFullName}/pulls/${pullNumber}`
+    }
+  );
+  const files = await paginateWithGitHubRateLimit(
+    (page) => octokit.pulls.listFiles({
+      owner,
+      repo,
+      pull_number: pullNumber,
+      per_page: 100,
+      page
+    }),
+    {
+      controller,
+      requestName: `GET /repos/${repoFullName}/pulls/${pullNumber}/files`
+    }
+  );
+  const reviews = await paginateWithGitHubRateLimit(
+    (page) => octokit.pulls.listReviews({
       owner,
       repo,
       pull_number: pullNumber,
-      per_page: 100
+      per_page: 100,
+      page
     }),
-    octokit.paginate(octokit.issues.listComments, {
+    {
+      controller,
+      requestName: `GET /repos/${repoFullName}/pulls/${pullNumber}/reviews`
+    }
+  );
+  const reviewComments = await paginateWithGitHubRateLimit(
+    (page) => octokit.pulls.listReviewComments({
+      owner,
+      repo,
+      pull_number: pullNumber,
+      per_page: 100,
+      page
+    }),
+    {
+      controller,
+      requestName: `GET /repos/${repoFullName}/pulls/${pullNumber}/comments`
+    }
+  );
+  const issueComments = await paginateWithGitHubRateLimit(
+    (page) => octokit.issues.listComments({
       owner,
       repo,
       issue_number: pullNumber,
-      per_page: 100
+      per_page: 100,
+      page
     }),
-    octokit.paginate(octokit.pulls.listCommits, { owner, repo, pull_number: pullNumber, per_page: 100 })
-  ]);
+    {
+      controller,
+      requestName: `GET /repos/${repoFullName}/issues/${pullNumber}/comments`
+    }
+  );
+  const commits = await paginateWithGitHubRateLimit(
+    (page) => octokit.pulls.listCommits({
+      owner,
+      repo,
+      pull_number: pullNumber,
+      per_page: 100,
+      page
+    }),
+    {
+      controller,
+      requestName: `GET /repos/${repoFullName}/pulls/${pullNumber}/commits`
+    }
+  );
   return {
     repo: repoFullName,
     number: pull.number,
@@ -4648,7 +4788,12 @@ async function fetchPullRequestDetailsConcurrently(options) {
         prNumber: pullNumber,
         detailConcurrency: options.detailConcurrency
       });
-      results[index] = await fetchPullRequestDetails(options.octokit, options.repo, pullNumber);
+      results[index] = await fetchPullRequestDetails(
+        options.octokit,
+        options.repo,
+        pullNumber,
+        options.controller
+      );
       completed += 1;
       options.onProgress?.({
         stage: "fetched_pull_request_details",
@@ -4674,10 +4819,18 @@ async function fetchMergedPullRequests(options) {
   const octokit = createGitHubClient(options.token);
   const limit = resolvePullRequestFetchLimit(options);
   const detailConcurrency = resolvePullRequestDetailConcurrency(options);
+  const rateLimitController = {
+    onRateLimit: (progress) => options.onProgress?.({
+      stage: "github_rate_limited",
+      repo: options.repo,
+      ...progress
+    })
+  };
   const sinceTime = options.since ? Date.parse(options.since) : void 0;
   const pullNumbers = [];
   let scannedPullRequests = 0;
   let reachedSinceBoundary = false;
+  let page = 1;
   options.onProgress?.({
     stage: "discovering_pull_requests",
     repo: options.repo,
@@ -4685,14 +4838,22 @@ async function fetchMergedPullRequests(options) {
     limit,
     since: options.since
   });
-  for await (const response of octokit.paginate.iterator(octokit.pulls.list, {
-    owner,
-    repo,
-    state: "closed",
-    sort: "updated",
-    direction: "desc",
-    per_page: 100
-  })) {
+  while (true) {
+    const response = await requestWithGitHubRateLimit(
+      () => octokit.pulls.list({
+        owner,
+        repo,
+        state: "closed",
+        sort: "updated",
+        direction: "desc",
+        per_page: 100,
+        page
+      }),
+      {
+        controller: rateLimitController,
+        requestName: `GET /repos/${options.repo}/pulls page ${page}`
+      }
+    );
     scannedPullRequests += response.data.length;
     for (const pull of response.data) {
       if (sinceTime && Date.parse(pull.updated_at) < sinceTime) {
@@ -4711,7 +4872,11 @@ async function fetchMergedPullRequests(options) {
       scannedPullRequests,
       matchedMergedPullRequests: pullNumbers.length
     });
-    if (reachedSinceBoundary || limit !== void 0 && pullNumbers.length >= limit) break;
+    const hasNextPage2 = String(response.headers.link ?? "").includes('rel="next"');
+    if (reachedSinceBoundary || limit !== void 0 && pullNumbers.length >= limit || !hasNextPage2) {
+      break;
+    }
+    page += 1;
   }
   options.onProgress?.({
     stage: "discovered_pull_requests",
@@ -4726,6 +4891,7 @@ async function fetchMergedPullRequests(options) {
     repo: options.repo,
     pullNumbers,
     detailConcurrency,
+    controller: rateLimitController,
     onProgress: options.onProgress
   });
 }
@@ -4974,6 +5140,7 @@ export {
   formatSearchHistory,
   getAnchorIndexHealth,
   getArchitectureContext,
+  getGitHubRateLimitDelayMs,
   getIndexStatus,
   getLastSyncTime,
   getSemanticStatus,
@@ -4986,6 +5153,7 @@ export {
   indexPullRequests,
   inferTestAwareness,
   initializeSchema,
+  isGitHubRateLimitError,
   isHardExcludedCodePath,
   isTestFilePath,
   loadCurrentCodeSnapshot,
@@ -4993,6 +5161,7 @@ export {
   mergeAnchorMcpConfig,
   normalizePullRequest,
   openAnchorDatabase,
+  paginateWithGitHubRateLimit,
   parseGitHubRemote,
   rankArchitecturePatterns,
   rankCodeChunks,
@@ -5004,6 +5173,7 @@ export {
   redactSecrets,
   redactedHistoricalText,
   replaceCodeIndex,
+  requestWithGitHubRateLimit,
   resolveGitHubToken,
   resolvePullRequestDetailConcurrency,
   resolvePullRequestFetchLimit,