megareviewer 0.1.0 → 0.2.0

package/dist/server.js

@@ -26356,7 +26356,7 @@ import { fileURLToPath } from "node:url";
 // package.json
 var package_default = {
   name: "megareviewer",
-  version: "0.1.0",
+  version: "0.2.0",
   description: "Guided code review for GitHub and Bitbucket Cloud pull requests: intent first, changes sorted by risk, decisions to approve.",
   keywords: [
     "code-review",
@@ -26510,6 +26510,7 @@ init_credentials();
 import { randomBytes } from "node:crypto";
 import { mkdirSync } from "node:fs";
 import { join as join4 } from "node:path";
+import { setTimeout as sleep2 } from "node:timers/promises";
 
 // node_modules/.pnpm/zod@4.6.7/node_modules/zod/v4/classic/external.js
 var external_exports = {};
@@ -47189,11 +47190,14 @@ var AnalysisError = external_exports.object({
   message: external_exports.string()
 }).readonly();
 var AnalysisStatus = external_exports.enum(["running", "complete", "failed"]);
+var FilesFetched = external_exports.object({ done: external_exports.number().int().nonnegative(), total: external_exports.number().int().nonnegative() }).readonly();
 var Analysis = external_exports.object({
   id: AnalysisId,
   key: AnalysisKey,
   status: AnalysisStatus,
   stages: external_exports.record(StageName, StageStatus).readonly(),
+  /** null until Collect starts fetching. Defaults for analyses stored before it existed. */
+  filesFetched: FilesFetched.nullable().default(null),
   pullRequest: PullRequest.nullable(),
   files: external_exports.array(FileChange).readonly(),
   tickets: external_exports.array(Ticket).readonly(),
@@ -47567,6 +47571,7 @@ var networkError = (error62) => {
   const cause = error62.cause instanceof Error ? error62.cause.message : error62.message;
   return hostError("network", `Can't reach Bitbucket: ${cause}`);
 };
+var timedOut = (timeoutMs) => hostError("network", `Bitbucket didn't answer within ${timeoutMs / 1e3} seconds`);
 var invalidResponse = (what, error62) => hostError(
   "invalid-response",
   `Unexpected response from Bitbucket for ${what}: ${typeof error62 === "string" ? error62 : external_exports.prettifyError(error62)}`
@@ -47578,10 +47583,12 @@ var API_ORIGIN = new URL(API).origin;
 var REDIRECTS = /* @__PURE__ */ new Set([301, 302, 303, 307, 308]);
 var MAX_REDIRECTS = 5;
 var MAX_PAGES = 10;
+var REQUEST_TIMEOUT_MS = 6e4;
 var createBitbucketClient = (options) => {
   const { logger } = options;
+  const timeoutMs = options.requestTimeoutMs ?? REQUEST_TIMEOUT_MS;
   const authorization = `Basic ${Buffer.from(`${options.username}:${options.password}`).toString("base64")}`;
-  const send = async (url2, accept, request2 = {}) => {
+  const send = async (url2, accept, request2, signal) => {
     let target = url2;
     for (let hop = 0; ; hop += 1) {
       const toApi = new URL(target).origin === API_ORIGIN;
@@ -47591,6 +47598,7 @@ var createBitbucketClient = (options) => {
         response = await options.fetch(target, {
           method: request2.method ?? "GET",
           redirect: "manual",
+          signal,
           headers: {
             accept,
             ...toApi ? { authorization } : {},
@@ -47609,17 +47617,28 @@ var createBitbucketClient = (options) => {
       target = new URL(location, target).href;
     }
   };
-  const receive = async (url2, accept, request2 = {}) => {
-    const response = await send(url2, accept, request2);
-    if (!response.ok || response.value.ok) return response;
-    const { status, headers } = response.value;
-    const body2 = await response.value.text();
-    return err2(fromErrorResponse({ status, headers, body: body2 }, request2.call ?? "read"));
+  const call = async (url2, accept, request2, read) => {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+      const response = await send(url2, accept, request2, controller.signal);
+      if (!response.ok) return response;
+      if (!response.value.ok) {
+        const { status, headers } = response.value;
+        const body2 = await response.value.text();
+        return err2(fromErrorResponse({ status, headers, body: body2 }, request2.call ?? "read"));
+      }
+      return await read(response.value);
+    } catch (error62) {
+      if (controller.signal.aborted) return err2(timedOut(timeoutMs));
+      if (error62 instanceof TypeError) return err2(networkError(error62));
+      throw error62;
+    } finally {
+      clearTimeout(timer);
+    }
   };
-  const json3 = async (what, schema, url2, request2 = {}) => {
-    const response = await receive(url2, "application/json", request2);
-    if (!response.ok) return response;
-    const text = await response.value.text();
+  const json3 = async (what, schema, url2, request2 = {}) => call(url2, "application/json", request2, async (response) => {
+    const text = await response.text();
     let data;
     try {
       data = text === "" ? null : JSON.parse(text);
@@ -47628,7 +47647,7 @@ var createBitbucketClient = (options) => {
     }
     const parsed = schema.safeParse(data);
     return parsed.success ? ok(parsed.data) : err2(invalidResponse(what, parsed.error));
-  };
+  });
   const pages = async (what, item, url2) => {
     const page = BitbucketPage(item);
     const items = [];
@@ -47648,11 +47667,15 @@ var createBitbucketClient = (options) => {
   return {
     json: json3,
     pages,
-    async text(url2, accept) {
-      const response = await receive(url2, accept);
-      if (!response.ok) return response;
-      return ok(new TextDecoder().decode(await response.value.arrayBuffer()));
-    }
+    text: (url2, accept) => (
+      // Bitbucket sends files and diffs in the repository's encoding, with no charset.
+      call(
+        url2,
+        accept,
+        {},
+        async (response) => ok(new TextDecoder().decode(await response.arrayBuffer()))
+      )
+    )
   };
 };
 
@@ -47713,6 +47736,7 @@ var VERDICT_ENDPOINTS = {
 // src/adapters/bitbucket/bitbucket-host.ts
 var SCANNED_REPOS = 50;
 var SCAN_CONCURRENCY = 5;
+var FILE_CONCURRENCY = 8;
 var PAGE_SIZE = 50;
 var FATAL = /* @__PURE__ */ new Set([
   "unauthorized",
@@ -47739,7 +47763,8 @@ var createBitbucketHost = (options) => {
     username: options.username,
     password: options.password,
     logger,
-    fetch: options.fetch ?? globalThis.fetch
+    fetch: options.fetch ?? globalThis.fetch,
+    requestTimeoutMs: options.requestTimeoutMs
   });
   let cachedUser = null;
   const currentUser = async () => {
@@ -47863,11 +47888,21 @@ var createBitbucketHost = (options) => {
         })
       );
     },
-    async getFile(repo, sha, path4) {
-      const encodedPath = path4.split("/").map(segment).join("/");
-      const content = await client.text(`${repoUrl(repo)}/src/${sha}/${encodedPath}`, "*/*");
-      if (!content.ok && content.error.kind === "not-found") return ok(null);
-      return content;
+    async getFiles(repo, files) {
+      const contents = await mapLimit(files, FILE_CONCURRENCY, async ({ sha, path: path4 }) => {
+        const encodedPath = path4.split("/").map(segment).join("/");
+        const content = await client.text(`${repoUrl(repo)}/src/${sha}/${encodedPath}`, "*/*");
+        if (!content.ok && (content.error.kind === "not-found" || content.error.kind === "too-large")) {
+          return ok(null);
+        }
+        return content;
+      });
+      const found = [];
+      for (const content of contents) {
+        if (!content.ok) return content;
+        found.push(content.value);
+      }
+      return ok(found);
     },
     async listPullRequests() {
       if (workspace === null) return ok([]);
@@ -52542,6 +52577,15 @@ var GitHubText = external_exports.union([
   external_exports.string(),
   external_exports.instanceof(ArrayBuffer).transform((buffer) => new TextDecoder().decode(buffer))
 ]);
+var GitHubObject = external_exports.object({
+  __typename: external_exports.string(),
+  text: external_exports.string().nullable().optional(),
+  isBinary: external_exports.boolean().nullable().optional(),
+  isTruncated: external_exports.boolean().optional()
+});
+var GitHubBlobs = external_exports.object({
+  repository: external_exports.record(external_exports.string(), GitHubObject.nullable())
+});
 var GitHubSearchResult = external_exports.object({
   total_count: external_exports.number().int().nonnegative(),
   incomplete_results: external_exports.boolean(),
@@ -52585,6 +52629,10 @@ var OctokitError = external_exports.object({
     data: external_exports.unknown()
   }).optional()
 });
+var GraphqlResponseError2 = external_exports.object({
+  name: external_exports.literal("GraphqlResponseError"),
+  errors: external_exports.array(external_exports.object({ type: external_exports.string().optional(), message: external_exports.string() })).min(1)
+});
 var hostError2 = (kind, message, retryAfterSeconds3 = null) => ({ kind, message, retryAfterSeconds: retryAfterSeconds3 });
 var header = (response, name2) => {
   const value = response.headers[name2];
@@ -52636,12 +52684,19 @@ var fromErrorResponse2 = (response, now) => {
   return hostError2("unexpected", `GitHub answered ${status}: ${message}`);
 };
 var fromOctokitError = (error62, now) => {
+  const graphql3 = GraphqlResponseError2.safeParse(error62);
+  if (graphql3.success) {
+    const { errors } = graphql3.data;
+    const message2 = `GitHub answered: ${errors.map((detail) => detail.message).join("; ")}`;
+    return hostError2(errors[0]?.type === "NOT_FOUND" ? "not-found" : "unexpected", message2);
+  }
   const parsed = OctokitError.safeParse(error62);
   if (!parsed.success) throw error62;
   const { response, message } = parsed.data;
   if (response === void 0) return hostError2("network", `Can't reach GitHub: ${message}`);
   return fromErrorResponse2(response, now);
 };
+var timedOut2 = (timeoutMs) => hostError2("network", `GitHub didn't answer within ${timeoutMs / 1e3} seconds`);
 var invalidResponse2 = (what, error62) => hostError2(
   "invalid-response",
   `Unexpected response from GitHub for ${what}: ${external_exports.prettifyError(error62)}`
@@ -52738,8 +52793,23 @@ var DIFF = "application/vnd.github.diff";
 var RAW = "application/vnd.github.raw";
 var MAX_COMMITS = 250;
 var SEARCH_PAGE_SIZE = 50;
+var FILES_PER_QUERY = 25;
+var REQUEST_TIMEOUT_MS2 = 6e4;
+var blobsQuery = (count2) => {
+  const indexes = Array.from({ length: count2 }, (_, i2) => i2);
+  const variables = indexes.map((i2) => `$e${i2}: String!`).join(", ");
+  const objects = indexes.map(
+    (i2) => `f${i2}: object(expression: $e${i2}) { __typename ... on Blob { text isBinary isTruncated } }`
+  ).join("\n    ");
+  return `query ($owner: String!, $repo: String!, ${variables}) {
+  repository(owner: $owner, name: $repo) {
+    ${objects}
+  }
+}`;
+};
 var createGitHubHost = (options) => {
   const { clock, logger } = options;
+  const timeoutMs = options.requestTimeoutMs ?? REQUEST_TIMEOUT_MS2;
   const octokit = new Octokit2({
     auth: options.token,
     userAgent: `megareviewer/${options.version}`,
@@ -52753,23 +52823,44 @@ var createGitHubHost = (options) => {
       error: (message) => logger.debug(message)
     }
   });
-  const read = async (what, schema, send) => {
+  const read = async (what, schema, send, { deadline = true } = {}) => {
+    const controller = deadline ? new AbortController() : null;
+    const timer = controller === null ? null : setTimeout(() => controller.abort(), timeoutMs);
     let data;
     try {
-      data = await send();
+      data = await send(controller === null ? {} : { signal: controller.signal });
     } catch (error62) {
+      if (controller?.signal.aborted) return err2(timedOut2(timeoutMs));
       return err2(fromOctokitError(error62, clock.now()));
+    } finally {
+      if (timer !== null) clearTimeout(timer);
     }
     const parsed = schema.safeParse(data);
     return parsed.success ? ok(parsed.data) : err2(invalidResponse2(what, parsed.error));
   };
+  const fileOverRest = async (repo, { sha, path: path4 }) => {
+    const encodedPath = path4.split("/").map(encodeURIComponent).join("/");
+    const content = await read(`${path4} at ${sha}`, GitHubText, async (request2) => {
+      return (await octokit.request(`GET /repos/{owner}/{repo}/contents/${encodedPath}`, {
+        owner: repo.owner,
+        repo: repo.repo,
+        ref: sha,
+        headers: { accept: RAW },
+        request: request2
+      })).data;
+    });
+    if (!content.ok && (content.error.kind === "not-found" || content.error.kind === "too-large")) {
+      return ok(null);
+    }
+    return content;
+  };
   let cachedUser = null;
   return {
     host: "github",
     async currentUser() {
       if (cachedUser !== null) return ok(cachedUser);
-      const user = await read("the current user", GitHubUser, async () => {
-        return (await octokit.request("GET /user")).data;
+      const user = await read("the current user", GitHubUser, async (request2) => {
+        return (await octokit.request("GET /user", { request: request2 })).data;
       });
       if (!user.ok) return user;
       cachedUser = toHostUser2(user.value);
@@ -52778,39 +52869,43 @@ var createGitHubHost = (options) => {
     async getPullRequest(ref) {
       const { owner, repo } = ref.repo;
       const pull_number = ref.number;
-      const pullRequest = await read("the pull request", GitHubPullRequest, async () => {
+      const pullRequest = await read("the pull request", GitHubPullRequest, async (request2) => {
         return (await octokit.request("GET /repos/{owner}/{repo}/pulls/{pull_number}", {
           owner,
           repo,
-          pull_number
+          pull_number,
+          request: request2
         })).data;
       });
       if (!pullRequest.ok) return pullRequest;
       const basehead = `${pullRequest.value.base.sha}...${pullRequest.value.head.sha}`;
       const [diff, comparison, commits] = await Promise.all([
-        read("the diff", GitHubText, async () => {
+        read("the diff", GitHubText, async (request2) => {
           return (await octokit.request("GET /repos/{owner}/{repo}/compare/{basehead}", {
             owner,
             repo,
             basehead,
-            headers: { accept: DIFF }
+            headers: { accept: DIFF },
+            request: request2
           })).data;
         }),
-        read("the merge base", GitHubComparison, async () => {
+        read("the merge base", GitHubComparison, async (request2) => {
           return (await octokit.request("GET /repos/{owner}/{repo}/compare/{basehead}", {
             owner,
             repo,
-            basehead
+            basehead,
+            request: request2
           })).data;
         }),
         read(
           "the commits",
           external_exports.array(GitHubCommit),
-          () => octokit.paginate("GET /repos/{owner}/{repo}/pulls/{pull_number}/commits", {
+          (request2) => octokit.paginate(octokit.rest.pulls.listCommits, {
             owner,
             repo,
             pull_number,
-            per_page: 100
+            per_page: 100,
+            request: request2
           })
         )
       ]);
@@ -52830,24 +52925,42 @@ var createGitHubHost = (options) => {
         })
       );
     },
-    async getFile(repo, sha, path4) {
-      const encodedPath = path4.split("/").map(encodeURIComponent).join("/");
-      const content = await read(`${path4} at ${sha}`, GitHubText, async () => {
-        return (await octokit.request(`GET /repos/{owner}/{repo}/contents/${encodedPath}`, {
-          owner: repo.owner,
-          repo: repo.repo,
-          ref: sha,
-          headers: { accept: RAW }
-        })).data;
-      });
-      if (!content.ok && content.error.kind === "not-found") return ok(null);
-      return content;
+    async getFiles(repo, files) {
+      const found = [];
+      for (let start3 = 0; start3 < files.length; start3 += FILES_PER_QUERY) {
+        const chunk = files.slice(start3, start3 + FILES_PER_QUERY);
+        const blobs = await read(
+          `${chunk.length} files`,
+          GitHubBlobs,
+          (request2) => octokit.graphql(blobsQuery(chunk.length), {
+            owner: repo.owner,
+            repo: repo.repo,
+            ...Object.fromEntries(chunk.map(({ sha, path: path4 }, i2) => [`e${i2}`, `${sha}:${path4}`])),
+            request: request2
+          })
+        );
+        if (!blobs.ok) return blobs;
+        for (const [i2, file2] of chunk.entries()) {
+          const object2 = blobs.value.repository[`f${i2}`];
+          if (object2 === null) {
+            found.push(null);
+          } else if (object2?.__typename === "Blob" && typeof object2.text === "string" && object2.isBinary !== true && object2.isTruncated !== true) {
+            found.push(object2.text);
+          } else {
+            const content = await fileOverRest(repo, file2);
+            if (!content.ok) return content;
+            found.push(content.value);
+          }
+        }
+      }
+      return ok(found);
     },
     async listPullRequests() {
-      const search = (qualifier) => read("the pull request search", GitHubSearchResult, async () => {
+      const search = (qualifier) => read("the pull request search", GitHubSearchResult, async (request2) => {
         return (await octokit.request("GET /search/issues", {
           q: `is:pr is:open archived:false ${qualifier}`,
-          per_page: SEARCH_PAGE_SIZE
+          per_page: SEARCH_PAGE_SIZE,
+          request: request2
         })).data;
       });
       const [reviewing, authored] = await Promise.all([
@@ -52880,24 +52993,33 @@ var createGitHubHost = (options) => {
       if (plan === null) return ok({ url: null, publishedAt: clock.now(), postedCommentIds: {} });
       const { owner, repo } = ref.repo;
       const pull_number = ref.number;
-      const review = await read("the new review", GitHubReview, async () => {
-        return (await octokit.request("POST /repos/{owner}/{repo}/pulls/{pull_number}/reviews", {
-          owner,
-          repo,
-          pull_number,
-          ...plan.request,
-          comments: [...plan.request.comments]
-        })).data;
-      });
+      const review = await read(
+        "the new review",
+        GitHubReview,
+        async () => {
+          return (await octokit.request("POST /repos/{owner}/{repo}/pulls/{pull_number}/reviews", {
+            owner,
+            repo,
+            pull_number,
+            ...plan.request,
+            comments: [...plan.request.comments]
+          })).data;
+        },
+        { deadline: false }
+      );
       if (!review.ok) return review;
       const reviewId = String(review.value.id);
       const hostComments = plan.comments.length === 0 ? ok([]) : await read(
         "the review's comments",
         external_exports.array(GitHubReviewComment),
-        () => octokit.paginate(
-          "GET /repos/{owner}/{repo}/pulls/{pull_number}/reviews/{review_id}/comments",
-          { owner, repo, pull_number, review_id: review.value.id, per_page: 100 }
-        )
+        (request2) => octokit.paginate(octokit.rest.pulls.listCommentsForReview, {
+          owner,
+          repo,
+          pull_number,
+          review_id: review.value.id,
+          per_page: 100,
+          request: request2
+        })
       );
       if (!hostComments.ok) {
         logger.warn("Couldn't read the review's comments back; recording the review's ID", {
@@ -56972,6 +57094,16 @@ var MIGRATIONS = [
     published_at TEXT NOT NULL,
     receipt TEXT NOT NULL           -- PublicationReceipt as JSON
   );
+  `,
+  `
+  CREATE TABLE file_contents (
+    host TEXT NOT NULL, owner TEXT NOT NULL, repo TEXT NOT NULL,
+    sha TEXT NOT NULL, path TEXT NOT NULL,
+    content TEXT,                   -- NULL: the file isn't at this commit
+    fetched_at TEXT NOT NULL,
+    PRIMARY KEY (host, owner, repo, sha, path)
+  );
+  CREATE INDEX file_contents_fetched_at ON file_contents (fetched_at);
   `
 ];
 var UserVersion = external_exports.object({ user_version: external_exports.number().int().nonnegative() });
@@ -57006,6 +57138,7 @@ var migrate = (db, migrations) => {
 };
 
 // src/adapters/sqlite/sqlite-analysis-store.ts
+var FILE_CACHE_DAYS = 30;
 var PayloadRow = external_exports.object({ id: external_exports.string(), payload: external_exports.string() });
 var VerdictRow = external_exports.object({
   decision_id: external_exports.string(),
@@ -57015,6 +57148,7 @@ var VerdictRow = external_exports.object({
 });
 var PostedCommentRow = external_exports.object({ comment_key: external_exports.string(), host_comment_id: external_exports.string() });
 var ReceiptRow = external_exports.object({ receipt: external_exports.string() });
+var FileRow = external_exports.object({ content: external_exports.string().nullable() });
 var parseJson3 = (text) => {
   try {
     return JSON.parse(text);
@@ -57075,6 +57209,20 @@ var openSqliteAnalysisStore = (options) => {
      ON CONFLICT (analysis_id, comment_key) DO UPDATE SET
        host_comment_id = excluded.host_comment_id, posted_at = excluded.posted_at`
   );
+  const selectFile = db.prepare(
+    `SELECT content FROM file_contents
+     WHERE host = ? AND owner = ? AND repo = ? AND sha = ? AND path = ?`
+  );
+  const upsertFile = db.prepare(
+    `INSERT INTO file_contents (host, owner, repo, sha, path, content, fetched_at)
+     VALUES (?, ?, ?, ?, ?, ?, ?)
+     ON CONFLICT (host, owner, repo, sha, path) DO UPDATE SET
+       content = excluded.content, fetched_at = excluded.fetched_at`
+  );
+  const pruned = db.prepare("DELETE FROM file_contents WHERE fetched_at < ?").run(new Date(Date.parse(clock.now()) - FILE_CACHE_DAYS * 24 * 60 * 60 * 1e3).toISOString());
+  if (Number(pruned.changes) > 0) {
+    logger.debug("Dropped old cached files", { count: Number(pruned.changes) });
+  }
   const selectPublication = db.prepare("SELECT receipt FROM publications WHERE analysis_id = ?");
   const upsertPublication = db.prepare(
     `INSERT INTO publications (analysis_id, url, published_at, receipt)
@@ -57184,6 +57332,13 @@ var openSqliteAnalysisStore = (options) => {
     async recordPublication(id, receipt) {
       upsertPublication.run(id, receipt.url, receipt.publishedAt, JSON.stringify(receipt));
     },
+    async get(repo, { sha, path: path4 }) {
+      const row = selectFile.get(repo.host, repo.owner, repo.repo, sha, path4);
+      return row === void 0 ? null : { content: FileRow.parse(row).content };
+    },
+    async put(repo, { sha, path: path4 }, content) {
+      upsertFile.run(repo.host, repo.owner, repo.repo, sha, path4, content, clock.now());
+    },
     close() {
       if (!open2) return;
       db.close();
@@ -61432,6 +61587,33 @@ var createAnalysisEvents = () => {
   };
 };
 
+// src/application/retries/retrying-git-host.ts
+var RETRY_DELAYS_MS = [1e3, 2e3, 4e3];
+var retryingGitHost = (host, { sleep: sleep3, logger }) => {
+  const retried = async (what, attempt) => {
+    let result = await attempt();
+    for (const delayMs of RETRY_DELAYS_MS) {
+      if (result.ok || result.error.kind !== "network") return result;
+      logger.warn(`Retrying ${what} after a network error`, {
+        host: host.host,
+        error: result.error.message,
+        delayMs
+      });
+      await sleep3(delayMs);
+      result = await attempt();
+    }
+    return result;
+  };
+  return {
+    host: host.host,
+    currentUser: () => retried("the current user", () => host.currentUser()),
+    getPullRequest: (ref) => retried("the pull request", () => host.getPullRequest(ref)),
+    getFiles: (repo, files) => retried(`${files.length} files`, () => host.getFiles(repo, files)),
+    listPullRequests: () => retried("the pull request list", () => host.listPullRequests()),
+    publish: (ref, publication, options) => host.publish(ref, publication, options)
+  };
+};
+
 // src/application/use-cases/errors.ts
 var appError = (code2, message) => ({ code: code2, message });
 var fromHostError = (error62) => appError(error62.kind === "not-found" ? "not-found" : "host-error", error62.message);
@@ -61456,9 +61638,11 @@ var mapLimit2 = async (items, limit2, map2) => {
 };
 
 // src/application/use-cases/collect.ts
-var FETCH_CONCURRENCY2 = 8;
+var FILES_PER_BATCH = 10;
+var BATCH_CONCURRENCY = 2;
 var MAX_FILE_BYTES = 1024 * 1024;
-var collect = async (deps, host, pullRequest) => {
+var collect = async (deps, host, pullRequest, onFilesFetched = async () => {
+}) => {
   const parsed = parseUnifiedDiff(pullRequest.diff);
   if (!parsed.ok) {
     const { line, message } = parsed.error;
@@ -61471,17 +61655,6 @@ var collect = async (deps, host, pullRequest) => {
     language: deps.parser.languageFor(file2.path)
   }));
   const { repo } = pullRequest.ref;
-  const fetchFile = async (sha, path4) => {
-    const result = await host.getFile(repo, sha, path4);
-    if (result.ok) return ok(fitsSize(result.value));
-    if (result.error.kind === "too-large") return ok(null);
-    return err2(
-      appError(
-        "host-error",
-        `Couldn't fetch ${path4} at ${sha.slice(0, 7)}: ${result.error.message}`
-      )
-    );
-  };
   const fetches = [
     { side: "attributes", sha: pullRequest.baseSha, path: ".gitattributes" },
     ...files.filter((file2) => !file2.binary).flatMap((file2) => [
@@ -61489,10 +61662,35 @@ var collect = async (deps, host, pullRequest) => {
       ...file2.status === "deleted" ? [] : [{ side: "head", sha: pullRequest.headSha, path: file2.path }]
     ])
   ];
-  const fetched = await mapLimit2(fetches, FETCH_CONCURRENCY2, async (fetch2) => ({
-    ...fetch2,
-    content: await fetchFile(fetch2.sha, fetch2.path)
-  }));
+  const cached2 = await Promise.all(fetches.map((fetch2) => deps.fileCache.get(repo, fetch2)));
+  const missing = fetches.map((fetch2, index) => ({ ...fetch2, index })).filter(({ index }) => cached2[index] === null);
+  const batches = Array.from(
+    { length: Math.ceil(missing.length / FILES_PER_BATCH) },
+    (_, i2) => missing.slice(i2 * FILES_PER_BATCH, (i2 + 1) * FILES_PER_BATCH)
+  );
+  const fetched = /* @__PURE__ */ new Map();
+  let done = fetches.length - missing.length;
+  await onFilesFetched({ done, total: fetches.length });
+  const failures = await mapLimit2(batches, BATCH_CONCURRENCY, async (batch) => {
+    const contents = await host.getFiles(repo, batch);
+    if (!contents.ok) return contents.error;
+    for (const [i2, file2] of batch.entries()) {
+      const content = contents.value[i2] ?? null;
+      fetched.set(file2.index, content);
+      if (content === null || fitsSize(content) !== null) {
+        await deps.fileCache.put(repo, file2, content);
+      }
+    }
+    done += batch.length;
+    await onFilesFetched({ done, total: fetches.length });
+    return null;
+  });
+  const failure2 = failures.find((error62) => error62 !== null) ?? null;
+  if (failure2 !== null) {
+    return err2(
+      appError("host-error", `Couldn't fetch the pull request's files: ${failure2.message}`)
+    );
+  }
   let gitattributes = null;
   const baseContents = /* @__PURE__ */ new Map();
   const headContents = /* @__PURE__ */ new Map();
@@ -61500,10 +61698,10 @@ var collect = async (deps, host, pullRequest) => {
     if (file2.status === "added") baseContents.set(file2.path, null);
     if (file2.status === "deleted") headContents.set(file2.path, null);
   }
-  for (const { side, path: path4, content } of fetched) {
-    if (!content.ok) return content;
-    if (side === "attributes") gitattributes = content.value;
-    else (side === "base" ? baseContents : headContents).set(path4, content.value);
+  for (const [index, { side, path: path4 }] of fetches.entries()) {
+    const content = fitsSize(cached2[index]?.content ?? fetched.get(index) ?? null);
+    if (side === "attributes") gitattributes = content;
+    else (side === "base" ? baseContents : headContents).set(path4, content);
   }
   return ok({
     files,
@@ -61828,6 +62026,10 @@ var createTracker = (deps, initial) => {
     }));
     deps.events.emit(analysis.id, { type: "updated", stage, status });
   };
+  const setFilesFetched = async (filesFetched) => {
+    await save({ filesFetched });
+    deps.events.emit(analysis.id, { type: "updated", stage: "collect", status: "running" });
+  };
   const finish = async (error62) => {
     await save({ status: error62 === null ? "complete" : "failed", error: error62 });
     deps.events.emit(analysis.id, { type: "done" });
@@ -61838,6 +62040,7 @@ var createTracker = (deps, initial) => {
     },
     save,
     setStage,
+    setFilesFetched,
     finish,
     fail: async (stage, failure2) => {
       await setStage(stage, "failed");
@@ -61864,7 +62067,7 @@ var runPipeline = async (deps, host, analysis, pullRequest) => {
 var stages = async (deps, host, pullRequest, tracker) => {
   const { model, logger } = deps;
   await tracker.setStage("collect", "running");
-  const collected = await collect(deps, host, pullRequest);
+  const collected = await collect(deps, host, pullRequest, tracker.setFilesFetched);
   if (!collected.ok) return tracker.fail("collect", collected.error);
   const { files, tickets, missingTicketKeys } = collected.value;
   await tracker.setStage("collect", "done", (analysis) => ({
@@ -61986,6 +62189,7 @@ var newAnalysis = (deps, key, pullRequest, createdAt) => ({
     decisions: "pending",
     tour: "pending"
   },
+  filesFetched: null,
   pullRequest,
   files: [],
   tickets: [],
@@ -62248,12 +62452,15 @@ var NO_ANTHROPIC_CREDENTIALS = "No Anthropic credentials found. Set ANTHROPIC_AP
 var createHosts = (config2, options) => {
   const { github, bitbucket } = config2;
   const { clock, logger } = options;
+  const retrying = (host) => retryingGitHost(host, { sleep: sleep2, logger });
   return {
     ...github !== null && {
-      github: createGitHubHost({ token: github.token, version: options.version, clock, logger })
+      github: retrying(
+        createGitHubHost({ token: github.token, version: options.version, clock, logger })
+      )
     },
     ...bitbucket !== null && {
-      bitbucket: createBitbucketHost({ ...bitbucket, clock, logger })
+      bitbucket: retrying(createBitbucketHost({ ...bitbucket, clock, logger }))
     }
   };
 };
@@ -62324,6 +62531,7 @@ var compose2 = async (config2, options) => {
           model: modelConfigured ? claude : unconfiguredModel(claude.version),
           modelConfigured,
           store,
+          fileCache: store,
           parser: await createTreeSitterCodeParser({ grammarsDir: options.grammarsDir }),
           clock,
           logger