@foss.global/codefeed 1.6.5 → 1.8.0

package/ts/index.ts

@@ -1,50 +1,310 @@
-import * as plugins from './codefeed.plugins.js';
+import * as plugins from './plugins.js';
 
 export class CodeFeed {
   private baseUrl: string;
   private token?: string;
-  private npmRegistry = new plugins.smartnpm.NpmRegistry();
-  private smartxmlInstance = new plugins.smartxml.SmartXml();
   private lastRunTimestamp: string;
-  private changelogContent: string;
-
-  constructor(baseUrl: string, token?: string, lastRunTimestamp?: string) {
+  private pageLimit = 50;
+  // Raw changelog content for the current repository
+  private changelogContent: string = '';
+  private npmRegistry?: plugins.TNpmRegistry;
+  // In-memory stateful cache of commits
+  private enableCache: boolean = false;
+  private cacheWindowMs?: number;
+  private cache: plugins.interfaces.ICommitResult[] = [];
+  private provider: plugins.interfaces.ICodeFeedProvider;
+  // enable or disable npm publishedOnNpm checks (true by default)
+  private enableNpmCheck: boolean = true;
+  // return only tagged commits (false by default)
+  private enableTaggedOnly: boolean = false;
+  // allow/deny filters
+  private orgAllowlist?: string[];
+  private orgDenylist?: string[];
+  private repoAllowlist?: string[]; // entries like "org/repo"
+  private repoDenylist?: string[]; // entries like "org/repo"
+  private untilTimestamp?: string; // optional upper bound on commit timestamps
+  private verbose?: boolean; // optional metrics logging
+
+  constructor(
+    baseUrl: string,
+    token?: string,
+    lastRunTimestamp?: string,
+    options?: plugins.interfaces.ICodeFeedOptions
+  ) {
     this.baseUrl = baseUrl;
     this.token = token;
     this.lastRunTimestamp =
-      lastRunTimestamp || new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString();
+      lastRunTimestamp ?? new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString();
+    // configure stateful caching
+    this.enableCache = options?.enableCache ?? false;
+    this.cacheWindowMs = options?.cacheWindowMs;
+    this.enableNpmCheck = options?.enableNpmCheck ?? true;
+    this.enableTaggedOnly = options?.taggedOnly ?? false;
+    this.orgAllowlist = options?.orgAllowlist;
+    this.orgDenylist = options?.orgDenylist;
+    this.repoAllowlist = options?.repoAllowlist;
+    this.repoDenylist = options?.repoDenylist;
+    this.untilTimestamp = options?.untilTimestamp;
+    this.verbose = options?.verbose ?? false;
+    this.cache = [];
+    this.provider = options?.provider ?? this.createDefaultProvider();
     console.log('CodeFeed initialized with last run timestamp:', this.lastRunTimestamp);
   }
 
   /**
-   * Load the changelog directly from the Gitea repository.
+   * Fetch all new commits (since lastRunTimestamp) across all orgs and repos.
    */
-  private async loadChangelogFromRepo(owner: string, repo: string): Promise<void> {
-    const url = `/api/v1/repos/${owner}/${repo}/contents/changelog.md`;
-    const headers: Record<string, string> = {};
-    if (this.token) {
-      headers['Authorization'] = `token ${this.token}`;
+  public async fetchAllCommitsFromInstance(): Promise<plugins.interfaces.ICommitResult[]> {
+    // Controlled concurrency with AsyncExecutionStack
+    const stack = new plugins.lik.AsyncExecutionStack();
+    stack.setNonExclusiveMaxConcurrency(20);
+    // determine since timestamp for this run (stateful caching)
+    let effectiveSince = this.lastRunTimestamp;
+    if (this.enableCache && this.cache.length > 0) {
+      // use newest timestamp in cache to fetch only tail
+      effectiveSince = this.cache.reduce(
+        (max, c) => (c.timestamp > max ? c.timestamp : max),
+        effectiveSince
+      );
+    }
+
+    // 1) get all organizations
+    let orgs = await this.provider.listOrganizations();
+    // apply allow/deny filters
+    if (this.orgAllowlist && this.orgAllowlist.length > 0) {
+      orgs = orgs.filter((o) => this.orgAllowlist!.includes(o));
+    }
+    if (this.orgDenylist && this.orgDenylist.length > 0) {
+      orgs = orgs.filter((o) => !this.orgDenylist!.includes(o));
     }
 
-    const response = await this.fetchFunction(url, { headers });
+    // 2) fetch repos per org in parallel
+    const repoLists = await Promise.all(
+      orgs.map((org) =>
+        stack.getNonExclusiveExecutionSlot(() => this.provider.listRepositoriesForOrg(org))
+      )
+    );
+    // flatten to [{ owner, name }]
+    let allRepos = orgs.flatMap((org, i) =>
+      repoLists[i].map((r) => ({ owner: org, name: r.name }))
+    );
+    // apply repo allow/deny filters using slug "org/repo"
+    if (this.repoAllowlist && this.repoAllowlist.length > 0) {
+      const allow = new Set(this.repoAllowlist.map((s) => s.toLowerCase()));
+      allRepos = allRepos.filter(({ owner, name }) => allow.has(`${owner}/${name}`.toLowerCase()));
+    }
+    if (this.repoDenylist && this.repoDenylist.length > 0) {
+      const deny = new Set(this.repoDenylist.map((s) => s.toLowerCase()));
+      allRepos = allRepos.filter(({ owner, name }) => !deny.has(`${owner}/${name}`.toLowerCase()));
+    }
 
-    if (!response.ok) {
-      console.error(
-        `Could not fetch CHANGELOG.md from ${owner}/${repo}: ${response.status} ${response.statusText}`
+    // 3) probe latest commit per repo and fetch full list only if new commits exist
+    const commitJobs = allRepos.map(({ owner, name }) =>
+      stack.getNonExclusiveExecutionSlot(async () => {
+        try {
+          const latestCommit = await this.provider.getLatestCommitForRepo(owner, name);
+          // If no commits or no new commits since last run, skip
+          if (
+            !latestCommit ||
+            new Date(latestCommit.commit.author.date).getTime() <= new Date(effectiveSince).getTime()
+          ) {
+            return { owner, name, commits: [] };
+          }
+          // 3b) Fetch commits since last run
+          const commits = await this.provider.listCommitsForRepo(
+            owner,
+            name,
+            effectiveSince
+          );
+          return { owner, name, commits };
+        } catch (e: any) {
+          console.error(`Failed to fetch commits for ${owner}/${name}:`, e.message);
+          return { owner, name, commits: [] };
+        }
+      })
+    );
+    const commitResults = await Promise.all(commitJobs);
+
+    // 4) build new commit entries with tagging, npm and changelog support
+    const newResults: plugins.interfaces.ICommitResult[] = [];
+    let reposWithNewCommits = 0;
+    for (const { owner, name, commits } of commitResults) {
+      // skip repos with no new commits
+      if (commits.length === 0) {
+        this.changelogContent = '';
+        continue;
+      }
+      reposWithNewCommits++;
+      // load changelog for this repo
+      this.changelogContent = await this.provider.readChangelogForRepo(owner, name);
+      // fetch tags for this repo
+      let taggedShas: Set<string>;
+      let tagNameBySha: Map<string, string>;
+      try {
+        const tagInfo = await this.provider.listTagsForRepo(owner, name);
+        taggedShas = tagInfo.shas;
+        tagNameBySha = tagInfo.map;
+      } catch (e: any) {
+        console.error(`Failed to fetch tags for ${owner}/${name}:`, e.message);
+        taggedShas = new Set<string>();
+        tagNameBySha = new Map<string, string>();
+      }
+      // fetch npm package info only if any new commits correspond to a tag
+      const hasTaggedCommit = commits.some((c) => taggedShas.has(c.sha));
+      let pkgInfo: { allVersions: Array<{ version: string }> } | null = null;
+      if (hasTaggedCommit && this.enableNpmCheck) {
+        try {
+          const npmRegistry = await this.getNpmRegistry();
+          pkgInfo = await npmRegistry.getPackageInfo(`@${owner}/${name}`);
+        } catch (e: any) {
+          console.error(`Failed to fetch package info for ${owner}/${name}:`, e.message);
+          pkgInfo = null;
+        }
+      }
+      // build commit entries
+      for (const c of commits) {
+        const isTagged = taggedShas.has(c.sha);
+        // derive version from tag name if present (strip leading 'v')
+        let versionFromTag: string | undefined;
+        if (isTagged) {
+          const tagName = tagNameBySha.get(c.sha);
+          if (tagName) {
+            versionFromTag = tagName.startsWith('v') ? tagName.substring(1) : tagName;
+          }
+        }
+        const publishedOnNpm = isTagged && pkgInfo && versionFromTag
+          ? pkgInfo.allVersions.some((v) => v.version === versionFromTag)
+          : false;
+        let changelogEntry: string | undefined;
+        if (this.changelogContent) {
+          if (versionFromTag) {
+            changelogEntry = this.getChangelogForVersion(versionFromTag);
+          }
+        }
+        // optionally enforce an upper bound on commit timestamps
+        if (this.untilTimestamp) {
+          const ts = new Date(c.commit.author.date).getTime();
+          if (ts > new Date(this.untilTimestamp).getTime()) {
+            continue;
+          }
+        }
+        newResults.push({
+          baseUrl: this.baseUrl,
+          org: owner,
+          repo: name,
+          timestamp: c.commit.author.date,
+          prettyAgoTime: plugins.smarttime.getMilliSecondsAsHumanReadableAgoTime(
+            new Date(c.commit.author.date).getTime()
+          ),
+          hash: c.sha,
+          commitMessage: c.commit.message,
+          tagged: isTagged,
+          publishedOnNpm,
+          changelog: changelogEntry,
+        });
+      }
+    }
+    // if caching is enabled, merge into in-memory cache and return full cache
+    if (this.enableCache) {
+      const existingHashes = new Set(this.cache.map((c) => c.hash));
+      const uniqueNew = newResults.filter((c) => !existingHashes.has(c.hash));
+      this.cache.push(...uniqueNew);
+      // trim commits older than window
+      if (this.cacheWindowMs !== undefined) {
+        const cutoff = Date.now() - this.cacheWindowMs;
+        this.cache = this.cache.filter((c) => new Date(c.timestamp).getTime() >= cutoff);
+      }
+      // advance lastRunTimestamp to now
+      this.lastRunTimestamp = new Date().toISOString();
+      // sort descending by timestamp
+      this.cache.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
+      // apply tagged-only filter if requested
+      if (this.enableTaggedOnly) {
+        return this.cache.filter((c) => c.tagged === true);
+      }
+      if (this.verbose) {
+        console.log(
+          `[CodeFeed] orgs=${orgs.length} repos=${allRepos.length} reposWithNew=${reposWithNewCommits} commits=${this.cache.length} (cached)`
+        );
+      }
+      return this.cache;
+    }
+    // no caching: apply tagged-only filter if requested
+    // sort and dedupe
+    const seen = new Set<string>();
+    const unique = newResults.filter((c) => {
+      if (seen.has(c.hash)) return false;
+      seen.add(c.hash);
+      return true;
+    });
+    unique.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
+    const result = this.enableTaggedOnly ? unique.filter((c) => c.tagged === true) : unique;
+    if (this.verbose) {
+      console.log(
+        `[CodeFeed] orgs=${orgs.length} repos=${allRepos.length} reposWithNew=${reposWithNewCommits} commits=${result.length}`
       );
-      this.changelogContent = '';
-      return;
     }
+    return result;
+  }
+
+  /**
+   * Load the changelog directly from the Gitea repository.
+   */
+  private createDefaultProvider(): plugins.interfaces.ICodeFeedProvider {
+    return {
+      listOrganizations: () => this.fetchAllOrganizations(),
+      listRepositoriesForOrg: (org) => this.fetchRepositoriesForOrg(org),
+      getLatestCommitForRepo: (owner, repo) => this.fetchLatestCommitForRepo(owner, repo),
+      listCommitsForRepo: (owner, repo, sinceTimestamp) => this.fetchRecentCommitsForRepo(owner, repo, sinceTimestamp),
+      listTagsForRepo: (owner, repo) => this.fetchTags(owner, repo),
+      readChangelogForRepo: async (owner, repo) => {
+        await this.loadChangelogFromRepo(owner, repo);
+        return this.changelogContent;
+      },
+    };
+  }
 
-    const data = await response.json();
-    if (!data.content) {
-      console.warn(`No content field found in response for ${owner}/${repo}/changelog.md`);
-      this.changelogContent = '';
-      return;
+  private createAuthHeaders(): Record<string, string> {
+    const headers: Record<string, string> = {};
+    if (this.token) {
+      headers.Authorization = `token ${this.token}`;
     }
+    return headers;
+  }
 
-    const decodedContent = Buffer.from(data.content, 'base64').toString('utf8');
-    this.changelogContent = decodedContent;
+  private async getNpmRegistry(): Promise<plugins.TNpmRegistry> {
+    if (!this.npmRegistry) {
+      const smartnpm = await plugins.getSmartnpm();
+      this.npmRegistry = new smartnpm.NpmRegistry();
+    }
+    return this.npmRegistry;
+  }
+
+  private async loadChangelogFromRepo(owner: string, repo: string): Promise<void> {
+    const headers = this.createAuthHeaders();
+    const candidates = [
+      'CHANGELOG.md',
+      'changelog.md',
+      'Changelog.md',
+      'docs/CHANGELOG.md',
+    ];
+    for (const path of candidates) {
+      const url = `/api/v1/repos/${owner}/${repo}/contents/${encodeURIComponent(path)}`;
+      const response = await this.fetchFunction(url, { headers });
+      if (!response.ok) {
+        continue;
+      }
+      try {
+        const data = await response.json();
+        if (data && data.content) {
+          this.changelogContent = Buffer.from(data.content, 'base64').toString('utf8');
+          return;
+        }
+      } catch {
+        // continue trying others
+      }
+    }
+    this.changelogContent = '';
   }
 
   /**
@@ -78,294 +338,137 @@ export class CodeFeed {
 
     return changelogLines.join('\n').trim();
   }
-
-  private async fetchAllOrganizations(): Promise<string[]> {
-    const url = `/api/v1/orgs`;
-    const response = await this.fetchFunction(url, {
-      headers: this.token ? { Authorization: `token ${this.token}` } : {},
-    });
-
-    if (!response.ok) {
-      throw new Error(`Failed to fetch organizations: ${response.statusText}`);
-    }
-
-    const data: { username: string }[] = await response.json();
-    return data.map((org) => org.username);
-  }
-
-  private async fetchOrgRssFeed(optionsArg: {
-    orgName: string;
-    repoName?: string;
-  }): Promise<any[]> {
-    let rssUrl: string;
-    if (optionsArg.orgName && !optionsArg.repoName) {
-      rssUrl = `/${optionsArg.orgName}.atom`;
-    } else if (optionsArg.orgName && optionsArg.repoName) {
-      rssUrl = `/${optionsArg.orgName}/${optionsArg.repoName}.atom`;
-    } else {
-      throw new Error('Invalid arguments provided to fetchOrgRssFeed.');
-    }
-
-    const response = await this.fetchFunction(rssUrl, {});
-    if (!response.ok) {
-      throw new Error(
-        `Failed to fetch RSS feed for organization ${optionsArg.orgName}/${optionsArg.repoName}: ${response.statusText}`
-      );
-    }
-
-    const rssText = await response.text();
-    const rssData = this.smartxmlInstance.parseXmlToObject(rssText);
-    return rssData.feed.entry || [];
-  }
-
-  private async hasNewActivity(optionsArg: {
-    orgName: string;
-    repoName?: string;
-  }): Promise<boolean> {
-    const entries = await this.fetchOrgRssFeed(optionsArg);
-
-    return entries.some((entry: any) => {
-      const updated = new Date(entry.updated);
-      return updated > new Date(this.lastRunTimestamp);
-    });
-  }
-
-  private async fetchAllRepositories(): Promise<plugins.interfaces.IRepository[]> {
+  /**
+   * Fetch all tags for a given repo and return the set of tagged commit SHAs
+   */
+  private async fetchTags(owner: string, repo: string): Promise<plugins.interfaces.ICodeFeedTagInfo> {
+    const taggedShas = new Set<string>();
+    const tagNameBySha = new Map<string, string>();
     let page = 1;
-    const allRepos: plugins.interfaces.IRepository[] = [];
-
     while (true) {
-      const url = `/api/v1/repos/search?limit=50&page=${page.toString()}`;
-
+      const url = `/api/v1/repos/${owner}/${repo}/tags?limit=${this.pageLimit}&page=${page}`;
       const resp = await this.fetchFunction(url, {
-        headers: this.token ? { Authorization: `token ${this.token}` } : {},
+        headers: this.createAuthHeaders(),
       });
-
       if (!resp.ok) {
-        throw new Error(`Failed to fetch repositories: ${resp.statusText}`);
+        console.error(`Failed to fetch tags for ${owner}/${repo}: ${resp.status} ${resp.statusText}`);
+        return { shas: taggedShas, map: tagNameBySha };
       }
-
-      const data: plugins.interfaces.IRepoSearchResponse = await resp.json();
-      allRepos.push(...data.data);
-
-      if (data.data.length < 50) {
-        break;
+      const data: plugins.interfaces.ITag[] = await resp.json();
+      if (data.length === 0) break;
+      for (const t of data) {
+        const sha = t.commit?.sha;
+        if (sha) {
+          taggedShas.add(sha);
+          if (t.name) tagNameBySha.set(sha, t.name);
+        }
       }
+      if (data.length < this.pageLimit) break;
       page++;
     }
+    return { shas: taggedShas, map: tagNameBySha };
+  }
 
-    return allRepos;
+  private async fetchLatestCommitForRepo(owner: string, repo: string): Promise<plugins.interfaces.ICommit | undefined> {
+    const resp = await this.fetchFunction(
+      `/api/v1/repos/${owner}/${repo}/commits?limit=1`,
+      { headers: this.createAuthHeaders() }
+    );
+    if (!resp.ok) {
+      throw new Error(`Probe failed for ${owner}/${repo}: ${resp.statusText}`);
+    }
+    const data: plugins.interfaces.ICommit[] = await resp.json();
+    return data[0];
   }
 
-  private async fetchTags(owner: string, repo: string): Promise<Set<string>> {
+  private async fetchAllOrganizations(): Promise<string[]> {
+    const headers = this.createAuthHeaders();
     let page = 1;
-    const tags: plugins.interfaces.ITag[] = [];
-
+    const orgs: string[] = [];
     while (true) {
-      const url = `/api/v1/repos/${owner}/${repo}/tags?limit=50&page=${page.toString()}`;
-
-      const resp = await this.fetchFunction(url, {
-        headers: this.token ? { Authorization: `token ${this.token}` } : {},
-      });
-
+      const resp = await this.fetchFunction(`/api/v1/orgs?limit=${this.pageLimit}&page=${page}`, { headers });
       if (!resp.ok) {
-        console.error(
-          `Failed to fetch tags for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url}`
-        );
-        throw new Error(`Failed to fetch tags for ${owner}/${repo}: ${resp.statusText}`);
-      }
-
-      const data: plugins.interfaces.ITag[] = await resp.json();
-      tags.push(...data);
-
-      if (data.length < 50) {
-        break;
+        throw new Error(`Failed to fetch organizations: ${resp.status} ${resp.statusText}`);
       }
+      const data: { username: string }[] = await resp.json();
+      if (data.length === 0) break;
+      orgs.push(...data.map((o) => o.username));
+      if (data.length < this.pageLimit) break;
       page++;
     }
+    return orgs;
+  }
 
-    const taggedCommitShas = new Set<string>();
-    for (const t of tags) {
-      if (t.commit?.sha) {
-        taggedCommitShas.add(t.commit.sha);
+  private async fetchRepositoriesForOrg(org: string): Promise<plugins.interfaces.IRepository[]> {
+    const headers = this.createAuthHeaders();
+    let page = 1;
+    const repos: plugins.interfaces.IRepository[] = [];
+    while (true) {
+      const resp = await this.fetchFunction(`/api/v1/orgs/${org}/repos?limit=${this.pageLimit}&page=${page}`, { headers });
+      if (!resp.ok) {
+        throw new Error(`Failed to fetch repositories for ${org}: ${resp.status} ${resp.statusText}`);
       }
+      const data: plugins.interfaces.IRepository[] = await resp.json();
+      if (data.length === 0) break;
+      repos.push(...data);
+      if (data.length < this.pageLimit) break;
+      page++;
     }
-
-    return taggedCommitShas;
+    return repos;
   }
 
   private async fetchRecentCommitsForRepo(
     owner: string,
-    repo: string
+    repo: string,
+    sinceTimestamp?: string
   ): Promise<plugins.interfaces.ICommit[]> {
-    const commitTimeframe = new Date(Date.now() - (7 * 24 * 60 * 60 * 1000));
+    const since = sinceTimestamp ?? this.lastRunTimestamp;
+    const headers = this.createAuthHeaders();
     let page = 1;
-    const recentCommits: plugins.interfaces.ICommit[] = [];
-
+    const commits: plugins.interfaces.ICommit[] = [];
     while (true) {
-      const url = `/api/v1/repos/${owner}/${repo}/commits?limit=50&page=${page.toString()}`;
-
-      const resp = await this.fetchFunction(url, {
-        headers: this.token ? { Authorization: `token ${this.token}` } : {},
-      });
-
+      const url = `/api/v1/repos/${owner}/${repo}/commits?since=${encodeURIComponent(since)}&limit=${this.pageLimit}&page=${page}`;
+      const resp = await this.fetchFunction(url, { headers });
       if (!resp.ok) {
-        console.error(
-          `Failed to fetch commits for ${owner}/${repo}: ${resp.status} ${resp.statusText} at ${url}`
-        );
-        throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.statusText}`);
+        throw new Error(`Failed to fetch commits for ${owner}/${repo}: ${resp.status} ${resp.statusText}`);
       }
-
       const data: plugins.interfaces.ICommit[] = await resp.json();
-      if (data.length === 0) {
-        break;
-      }
-
-      for (const commit of data) {
-        const commitDate = new Date(commit.commit.author.date);
-        if (commitDate > commitTimeframe) {
-          recentCommits.push(commit);
-        } else {
-          return recentCommits;
-        }
-      }
-
+      if (data.length === 0) break;
+      commits.push(...data);
+      if (data.length < this.pageLimit) break;
       page++;
     }
-
-    return recentCommits;
+    return commits;
   }
 
-  public async fetchAllCommitsFromInstance(): Promise<plugins.interfaces.ICommitResult[]> {
-    const orgs = await this.fetchAllOrganizations();
-    console.log(`Found ${orgs.length} organizations`);
-    let allCommits: plugins.interfaces.ICommitResult[] = [];
-
-    for (const orgName of orgs) {
-      console.log(`Checking activity for organization: ${orgName}`);
-
+  public async fetchFunction(
+    urlArg: string,
+    optionsArg: RequestInit = {}
+  ): Promise<Response> {
+    const maxAttempts = 4;
+    let attempt = 0;
+    let lastError: any;
+    while (attempt < maxAttempts) {
       try {
-        const hasActivity = await this.hasNewActivity({
-          orgName,
-        });
-        if (!hasActivity) {
-          console.log(`No new activity for organization: ${orgName}`);
-          continue;
-        }
-      } catch (error: any) {
-        console.error(`Error fetching activity for organization ${orgName}:`, error.message);
-        continue;
-      }
-
-      console.log(`New activity detected for organization: ${orgName}. Processing repositories...`);
-
-      const repos = await this.fetchAllRepositories();
-      for (const r of repos.filter((repo) => repo.owner.login === orgName)) {
-        try {
-          const hasActivity = await this.hasNewActivity({
-            orgName,
-            repoName: r.name,
-          });
-          if (!hasActivity) {
-            console.log(`No new activity for repository: ${orgName}/${r.name}`);
-            continue;
-          }
-        } catch (error: any) {
-          console.error(
-            `Error fetching activity for repository ${orgName}/${r.name}:`,
-            error.message
-          );
+        const resp = await fetch(`${this.baseUrl}${urlArg}`, optionsArg);
+        // retry on 429 and 5xx
+        if (resp.status === 429 || resp.status >= 500) {
+          const retryAfter = Number(resp.headers.get('retry-after'));
+          const backoffMs = retryAfter
+            ? retryAfter * 1000
+            : Math.min(32000, 1000 * Math.pow(2, attempt)) + Math.floor(Math.random() * 250);
+          await new Promise((r) => setTimeout(r, backoffMs));
+          attempt++;
           continue;
         }
-
-        const org = r.owner.login;
-        const repo = r.name;
-        console.log(`Processing repository ${org}/${repo}`);
-
-        try {
-          const taggedCommitShas = await this.fetchTags(org, repo);
-          const commits = await this.fetchRecentCommitsForRepo(org, repo);
-
-          // Load the changelog from this repo.
-          await this.loadChangelogFromRepo(org, repo);
-
-          const commitResults = commits.map((c) => {
-            const commit: plugins.interfaces.ICommitResult = {
-              baseUrl: this.baseUrl,
-              org,
-              repo,
-              timestamp: c.commit.author.date,
-              prettyAgoTime: plugins.smarttime.getMilliSecondsAsHumanReadableAgoTime(
-                new Date(c.commit.author.date).getTime()
-              ),
-              hash: c.sha,
-              commitMessage: c.commit.message,
-              tagged: taggedCommitShas.has(c.sha),
-              publishedOnNpm: false,
-              changelog: undefined,
-            };
-            return commit;
-          });
-
-          if (commitResults.length > 0) {
-            try {
-              const packageInfo = await this.npmRegistry.getPackageInfo(`@${org}/${repo}`);
-              for (const commitResult of commitResults.filter((c) => c.tagged)) {
-                const versionCandidate = commitResult.commitMessage.replace('\n', '').trim();
-                const correspondingVersion = packageInfo.allVersions.find((versionArg) => {
-                  return versionArg.version === versionCandidate;
-                });
-                if (correspondingVersion) {
-                  commitResult.publishedOnNpm = true;
-                }
-              }
-            } catch (error: any) {
-              console.error(`Failed to fetch package info for ${org}/${repo}:`, error.message);
-            }
-
-            try {
-              for (const commitResult of commitResults.filter((c) => c.tagged)) {
-                const versionCandidate = commitResult.commitMessage.replace('\n', '').trim();
-                const changelogEntry = this.getChangelogForVersion(versionCandidate);
-                if (changelogEntry) {
-                  commitResult.changelog = changelogEntry;
-                }
-              }
-            } catch (error: any) {
-              console.error(`Failed to fetch changelog info for ${org}/${repo}:`, error.message);
-            }
-          }
-
-          allCommits.push(...commitResults);
-        } catch (error: any) {
-          console.error(`Skipping repository ${org}/${repo} due to error:`, error.message);
-        }
+        return resp;
+      } catch (e: any) {
+        lastError = e;
+        const backoffMs = Math.min(32000, 1000 * Math.pow(2, attempt)) + Math.floor(Math.random() * 250);
+        await new Promise((r) => setTimeout(r, backoffMs));
+        attempt++;
       }
     }
-
-    console.log(`Processed ${allCommits.length} commits in total.`);
-
-    allCommits = allCommits
-      .filter((commitArg) => commitArg.tagged)
-      .sort((a, b) => b.timestamp.localeCompare(a.timestamp));
-
-    console.log(`Filtered to ${allCommits.length} commits with tagged statuses.`);
-
-    for (const c of allCommits) {
-      console.log(`      ==========================================================================
-      ${c.prettyAgoTime} ago:
-      ${c.org}/${c.repo}
-      ${c.commitMessage}
-      Published on npm: ${c.publishedOnNpm}
-      ${c.changelog ? `Changelog:\n${c.changelog}\n` : ''}
-    `);
-    }
-
-    return allCommits;
-  }
-
-  public async fetchFunction(urlArg: string, optionsArg: RequestInit): Promise<Response> {
-    const response = await fetch(`${this.baseUrl}${urlArg}`, optionsArg);
-    return response;
+    throw new Error(`fetchFunction failed after retries for ${urlArg}: ${lastError?.message ?? 'unknown error'}`);
   }
 }