@cat-factory/executor-harness 1.58.0 → 1.62.0

package/src/git.ts

@@ -3,7 +3,7 @@ import { appendFile, chmod, mkdtemp, rm, writeFile } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { promisify } from 'node:util'
-import type { BootstrapTargetSpec, PrSpec, RepoSpec } from './job.js'
+import type { BootstrapTargetSpec, RepoSpec } from './job.js'
 import { pathExists } from './fs-utils.js'
 import { redactSecrets } from './redact.js'
 import { loadRunnerLimits } from './runner.js'
@@ -174,51 +174,6 @@ export function describeGitFailure(stderr: string): string | undefined {
   return undefined
 }
 
-/**
- * Classify a PR/MR-open REST failure by its HTTP status into an actionable remedy, else
- * undefined (an unmapped status keeps just the raw `Failed to open … (HTTP n)` line). Like
- * {@link describeGitFailure} this only APPENDS a cause + fix — the raw status line is
- * load-bearing detail and stays. `provider` tailors the scope/permission wording (GitHub App
- * Pull-requests permission / `repo` PAT scope vs GitLab `api` scope) and the noun (pull
- * request vs merge request). Pure, so it is unit-tested per status.
- */
-export function describePrOpenFailure(
-  status: number,
-  provider: 'github' | 'gitlab',
-): string | undefined {
-  const noun = provider === 'gitlab' ? 'merge request' : 'pull request'
-  if (status === 401) {
-    return (
-      `The credential was rejected while opening the ${noun}. The GitHub App installation token ` +
-      '(or, in local mode, the GITHUB_PAT) is most likely expired, rotated, or revoked — reconnect ' +
-      'the GitHub App for the workspace (or regenerate the PAT), then retry.'
-    )
-  }
-  if (status === 403) {
-    const scope =
-      provider === 'gitlab'
-        ? 'the GitLab token needs the `api` scope and Developer+ access to the project'
-        : 'the GitHub App needs the "Pull requests: write" permission (or the PAT the `repo` scope) and write access to the repository'
-    return `The credential lacks permission to open a ${noun}: ${scope}. Grant it, then retry.`
-  }
-  if (status === 404) {
-    return (
-      `The repository could not be found while opening the ${noun} — it may have been deleted, ` +
-      'renamed, or made private, or the credential can no longer see it. Confirm the repo and the ' +
-      "credential's access to it, then retry."
-    )
-  }
-  if (status === 422 || status === 400) {
-    return (
-      `GitHub/GitLab rejected the ${noun} as invalid. Usually the head or base branch does not ` +
-      'exist, the two branches are identical (nothing to compare), or the base branch is protected ' +
-      'against direct PRs. Check the branch names and that the head has commits ahead of the base, ' +
-      'then retry.'
-    )
-  }
-  return undefined
-}
-
 /**
  * Wrap a git failure into a credential-scrubbed {@link HarnessFailure}('git') with an ACCURATE
  * message. Three cases the old bare "Command failed: git …" collapsed together:
@@ -648,6 +603,46 @@ export async function branchAheadOfBase(
   }
 }
 
+/**
+ * The files `commitish` changes relative to its merge base with the PR base branch — i.e.
+ * everything the work branch has added on top of base, `git diff --name-only <base>...<commitish>`.
+ *
+ * The BUGFIX REPRODUCTION PROOF uses this to answer the one question that decides whether a GREEN
+ * pre-fix tree means anything: does that tree ALREADY carry non-test work committed on this
+ * branch? A resumed run's `baseSha` is whatever the branch tip was when this pass started, which
+ * in the designed flow is the reproduction step's test commit — but after an eviction it is this
+ * same coder step's own interrupted work, fix included. Reporting "the check passed before your
+ * change, so it does not demonstrate the defect" in that case is simply false.
+ *
+ * `undefined` means "could not determine" (a shallow clone with no reachable merge base, a fetch
+ * failure, an unknown ref), never an empty list: the caller must degrade to its prior behaviour
+ * rather than read a failed probe as "the tree is clean".
+ *
+ * NUL-delimited so a path containing a newline (legal in git) cannot split into two entries.
+ */
+export async function changedFilesSinceBase(
+  dir: string,
+  baseBranch: string,
+  ghToken: string,
+  commitish: string,
+  signal?: AbortSignal,
+): Promise<string[] | undefined> {
+  try {
+    await git(['fetch', 'origin', `+refs/heads/${baseBranch}:refs/cat-factory/base`], {
+      cwd: dir,
+      signal,
+      env: await authEnv(ghToken),
+    })
+    const out = await git(['diff', '--name-only', '-z', `refs/cat-factory/base...${commitish}`], {
+      cwd: dir,
+      signal,
+    })
+    return out.split('\0').filter((p) => p !== '')
+  } catch {
+    return undefined
+  }
+}
+
 /**
  * Whether the checked-out branch has a real, examinable diff against
  * `origin/<baseBranch>` — i.e. the base branch's remote-tracking ref exists (so the
@@ -708,6 +703,99 @@ export async function headCommit(dir: string, signal?: AbortSignal): Promise<str
   return (await git(['rev-parse', 'HEAD'], { cwd: dir, signal })).trim()
 }
 
+/**
+ * Add a DETACHED worktree of `commitish` at `worktreePath`, sharing `dir`'s object database.
+ *
+ * The bugfix reproduction proof runs the declared check against two trees of the SAME clone (the
+ * pre-fix tree and the final tree), so a worktree is the only mechanism that gets both without a
+ * second clone, a second fetch, or disturbing the agent's own checkout — which must stay exactly
+ * as the agent left it, since the push and the PR come off it.
+ *
+ * `--detach` (rather than a branch) is deliberate: a worktree that claimed a branch would collide
+ * with the work branch checked out in `dir`, and nothing here ever commits.
+ *
+ * `worktreePath` is expected to live OUTSIDE the checkout (a per-job temp root), so the worktree's
+ * `.git` pointer file can never be swept into the agent's commit by a broad `git add -A`.
+ */
+export async function addWorktree(
+  dir: string,
+  worktreePath: string,
+  commitish: string,
+  signal?: AbortSignal,
+): Promise<void> {
+  await git(['worktree', 'add', '--detach', worktreePath, commitish], { cwd: dir, signal })
+}
+
+/**
+ * Remove a worktree previously added by {@link addWorktree} and prune the stale administrative
+ * entry, never throwing: teardown is bookkeeping, and a run whose PROOF succeeded must not fail
+ * because a temp directory could not be cleaned up. The caller still deletes the temp root, so a
+ * failure here leaks only a `.git/worktrees/<name>` record inside a container that is about to be
+ * destroyed anyway.
+ */
+export async function removeWorktree(
+  dir: string,
+  worktreePath: string,
+  signal?: AbortSignal,
+): Promise<void> {
+  try {
+    await git(['worktree', 'remove', '--force', worktreePath], { cwd: dir, signal })
+  } catch {
+    // Fall through to the prune, which cleans up the record even when the directory is gone.
+  }
+  try {
+    await git(['worktree', 'prune'], { cwd: dir, signal })
+  } catch {
+    // Best-effort by design (see the doc comment).
+  }
+}
+
+/**
+ * Which of `paths` actually exist in `commitish`'s tree. Used by the reproduction proof to tell a
+ * DECLARED test file that was committed from one that only ever existed as an untracked working-
+ * tree file: the proof runs against committed trees, so an unadded test is invisible to it — and
+ * equally invisible to the push, which is the point worth telling the agent about rather than
+ * reporting a verdict computed without the reproduction in it.
+ *
+ * Returns the input order/spelling of the paths that matched, so the caller can diff against its
+ * declared list to name the missing ones verbatim.
+ */
+export async function pathsPresentAtCommit(
+  dir: string,
+  commitish: string,
+  paths: readonly string[],
+  signal?: AbortSignal,
+): Promise<string[]> {
+  if (paths.length === 0) return []
+  const out = await git(['ls-tree', '-r', '--name-only', '-z', commitish, '--', ...paths], {
+    cwd: dir,
+    signal,
+  })
+  // NUL-delimited so a path containing a newline (legal in git) can't split into two entries.
+  const present = new Set(out.split('\0').filter((p) => p !== ''))
+  return paths.filter((p) => present.has(p))
+}
+
+/**
+ * Check `paths` out of `commitish` into `dir`'s working tree (and index), leaving every other file
+ * untouched.
+ *
+ * This is how the reproduction's declared TEST files are placed onto the pre-fix worktree, and the
+ * narrowness is the whole safety property: a whole-tree checkout would drag the FIX across too and
+ * green the base, manufacturing a "the test does not capture the defect" verdict out of a
+ * perfectly good reproduction. Only the paths the caller has already sanitized are passed, and
+ * `--` stops any of them being read as a revision.
+ */
+export async function checkoutPathsFrom(
+  dir: string,
+  commitish: string,
+  paths: readonly string[],
+  signal?: AbortSignal,
+): Promise<void> {
+  if (paths.length === 0) return
+  await git(['checkout', commitish, '--', ...paths], { cwd: dir, signal })
+}
+
 /** Stage everything and commit; returns false when there was nothing to commit. */
 export async function commitAll(
   dir: string,
@@ -963,342 +1051,3 @@ export async function reinitAndPush(opts: {
     env: await authEnv(opts.ghToken),
   })
 }
-
-export interface OpenPullRequestOptions {
-  owner: string
-  name: string
-  ghToken: string
-  head: string
-  base: string
-  pr: PrSpec
-  apiBase?: string
-  /**
-   * The repo's clone URL. Used (when {@link provider} is absent) to detect the provider and,
-   * for GitLab, to derive the REST base + project path from its host — so the harness opens a
-   * GitLab **merge request** rather than POSTing to GitHub's pulls API. Absent ⇒ GitHub.
-   */
-  cloneUrl?: string
-  /**
-   * The VCS provider, when the dispatcher knows it (the server derives it from the configured
-   * source-control backend and sets `repo.provider`). AUTHORITATIVE — it overrides host
-   * inference — so a self-managed GitLab on an arbitrarily-named host (e.g. `git.acme.com`,
-   * which {@link inferVcsProvider} can't recognise) still opens a merge request instead of
-   * being misrouted to GitHub's API. Absent ⇒ inferred from {@link cloneUrl}'s host.
-   */
-  provider?: 'github' | 'gitlab'
-  signal?: AbortSignal
-}
-
-/**
- * The VCS host a clone URL points at. The harness is otherwise provider-agnostic (its git
- * auth is a host-neutral GIT_ASKPASS credential), but the "open the PR/MR" REST call is not:
- * GitHub and GitLab have different endpoints, so infer which to call from the host. GitHub is
- * the default; a host of `gitlab.com` or one in the `gitlab.*` / `*.gitlab.*` family (covering
- * self-managed instances named that way) is treated as GitLab.
- */
-export function inferVcsProvider(cloneUrl: string): 'github' | 'gitlab' {
-  let host = ''
-  try {
-    host = new URL(cloneUrl).host.toLowerCase()
-  } catch {
-    return 'github'
-  }
-  if (host === 'gitlab.com' || host.startsWith('gitlab.') || host.includes('.gitlab.')) {
-    return 'gitlab'
-  }
-  return 'github'
-}
-
-/** The GitLab REST v4 base for a clone URL's host, e.g. `https://gitlab.com/api/v4`. */
-export function gitlabApiBaseFromCloneUrl(cloneUrl: string): string {
-  const u = new URL(cloneUrl)
-  return `${u.protocol}//${u.host}/api/v4`
-}
-
-/**
- * The URL-encoded GitLab project path from a clone URL — the full namespace path (so subgroups
- * survive), with the trailing `.git` stripped, e.g.
- * `https://gitlab.com/group/sub/proj.git` → `group%2Fsub%2Fproj`.
- */
-export function gitlabProjectPath(cloneUrl: string): string {
-  const path = new URL(cloneUrl).pathname.replace(/^\/+/, '').replace(/\.git$/, '')
-  return encodeURIComponent(path)
-}
-
-/** The abort reason as an Error (the watchdog aborts with one), or a generic fallback. */
-function abortError(signal: AbortSignal): Error {
-  return signal.reason instanceof Error ? signal.reason : new Error('aborted')
-}
-
-/** Whether a thrown fetch error is an AbortError (caller-initiated, never retried). */
-function isAbortError(err: unknown): boolean {
-  return err instanceof Error && err.name === 'AbortError'
-}
-
-/**
- * Parse a `Retry-After` header into ms, bounded so it can't stall the job. Accepts BOTH
- * forms the spec allows: integer delay-seconds (`120`) and an HTTP-date (`Wed, 21 Oct 2026
- * 07:28:00 GMT`); the latter is turned into a delay from now. A past/zero/unparseable value
- * yields undefined so the caller falls back to exponential backoff.
- */
-function retryAfterMs(res: Response): number | undefined {
-  const raw = res.headers.get('retry-after')
-  if (!raw) return undefined
-  const secs = Number(raw)
-  if (Number.isFinite(secs)) {
-    return secs > 0 ? Math.min(secs * 1000, MAX_RETRY_AFTER_MS) : undefined
-  }
-  const at = Date.parse(raw)
-  if (Number.isNaN(at)) return undefined
-  const ms = at - Date.now()
-  return ms > 0 ? Math.min(ms, MAX_RETRY_AFTER_MS) : undefined
-}
-
-/** Sleep `ms`, rejecting immediately (with the abort reason) if `signal` aborts meanwhile. */
-function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
-  return new Promise((resolve, reject) => {
-    if (signal?.aborted) return reject(abortError(signal))
-    const onAbort = (): void => {
-      clearTimeout(timer)
-      reject(abortError(signal as AbortSignal))
-    }
-    const timer = setTimeout(() => {
-      signal?.removeEventListener('abort', onAbort)
-      resolve()
-    }, ms)
-    signal?.addEventListener('abort', onAbort, { once: true })
-  })
-}
-
-const MAX_RETRY_AFTER_MS = 8_000
-const RETRY_BASE_MS = 500
-const RETRY_MAX_DELAY_MS = 4_000
-
-/**
- * Run a single HTTP request with bounded retry for TRANSIENT failures, so a momentary
- * upstream blip (a 5xx, a 429 rate-limit, or a dropped connection) no longer fails an
- * otherwise-complete run on its very last step (opening the PR/MR). Up to 3 attempts
- * (2 retries) with exponential backoff + jitter (honoring a `Retry-After` on a 429),
- * every wait abort-aware so the inactivity/max-duration watchdog still cancels promptly.
- *
- * ONLY transient failures retry: a `>=500`/`429` response, or a network-level fetch
- * rejection. A 4xx (incl. the 422/409 "already exists" the callers treat as success) is
- * returned to the caller unretried, and a caller abort is rethrown at once. The response
- * body is never read here, so the caller's existing status handling is unchanged.
- */
-async function withApiRetry(
-  fn: () => Promise<Response>,
-  opts: { signal?: AbortSignal; attempts?: number } = {},
-): Promise<Response> {
-  const maxAttempts = opts.attempts ?? 3
-  let lastError: unknown
-  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
-    if (opts.signal?.aborted) throw abortError(opts.signal)
-    let res: Response | undefined
-    try {
-      res = await fn()
-    } catch (err) {
-      // A caller/watchdog abort is terminal; a network error is transient → retry.
-      if (isAbortError(err) || opts.signal?.aborted) throw err
-      lastError = err
-    }
-    if (res) {
-      const transient = res.status >= 500 || res.status === 429
-      if (!transient || attempt >= maxAttempts) return res
-      const after = retryAfterMs(res)
-      // Discard the unread body before retrying so the connection can be reused.
-      await res.body?.cancel().catch(() => {})
-      await abortableDelay(after ?? backoffMs(attempt), opts.signal)
-      continue
-    }
-    if (attempt >= maxAttempts) break
-    await abortableDelay(backoffMs(attempt), opts.signal)
-  }
-  // Exhausted on a network-level rejection (no HTTP response): an upstream API failure.
-  const message =
-    lastError instanceof Error ? lastError.message : 'API request failed after retries'
-  throw new HarnessFailure('api', redactSecrets(message))
-}
-
-/** Exponential backoff (base 500ms, capped 4s) with up to 25% positive jitter. */
-function backoffMs(attempt: number): number {
-  const base = Math.min(RETRY_MAX_DELAY_MS, RETRY_BASE_MS * 2 ** (attempt - 1))
-  return base + Math.floor(base * 0.25 * Math.random())
-}
-
-/**
- * Open a PR (GitHub) or merge request (GitLab) for the pushed branch; returns its web URL.
- * The provider is chosen from the EXPLICIT `opts.provider` when the dispatcher set it,
- * falling back to host inference from the clone URL only when it didn't — so a self-managed
- * GitLab whose host isn't named `gitlab.*` still opens an MR instead of being misrouted to
- * GitHub's API. The GitHub path is unchanged.
- */
-export async function openPullRequest(opts: OpenPullRequestOptions): Promise<string | null> {
-  const provider = opts.provider ?? (opts.cloneUrl ? inferVcsProvider(opts.cloneUrl) : 'github')
-  if (provider === 'gitlab') {
-    if (!opts.cloneUrl) {
-      throw new Error('Cannot open a GitLab merge request without the repo clone URL')
-    }
-    return openGitLabMergeRequest({ ...opts, cloneUrl: opts.cloneUrl })
-  }
-  const apiBase = opts.apiBase ?? 'https://api.github.com'
-  const path = `${encodeURIComponent(opts.owner)}/${encodeURIComponent(opts.name)}`
-  const res = await withApiRetry(
-    () =>
-      fetch(`${apiBase}/repos/${path}/pulls`, {
-        method: 'POST',
-        headers: {
-          authorization: `Bearer ${opts.ghToken}`,
-          accept: 'application/vnd.github+json',
-          'user-agent': 'cat-factory-executor',
-          'x-github-api-version': '2022-11-28',
-          'content-type': 'application/json',
-        },
-        body: JSON.stringify({
-          title: opts.pr.title,
-          head: opts.head,
-          base: opts.base,
-          body: opts.pr.body,
-        }),
-        // Bound on the watchdog so a hung GitHub call can't stall the job.
-        ...(opts.signal ? { signal: opts.signal } : {}),
-      }),
-    { signal: opts.signal },
-  )
-  if (!res.ok) {
-    const detail = await res.text().catch(() => '')
-    // A resumed run pushes to a branch that already has an open PR; GitHub answers
-    // 422 "A pull request already exists". That's success for us — return the
-    // existing PR's url rather than failing the resumed run.
-    if (res.status === 422 && /pull request already exists/i.test(detail)) {
-      const existing = await findOpenPullRequestUrl(opts)
-      if (existing) return existing
-    }
-    // The head branch has nothing ahead of base ("No commits between <base> and <head>").
-    // That is not an API failure — there is simply nothing to open a PR for (e.g. a resumed
-    // branch whose earlier PR was merged with a merge commit, leaving the branch reachable
-    // from base). Signal it with null so the caller records a clean no-op instead of failing
-    // the run with GitHub's opaque 422.
-    if (res.status === 422 && /no commits between/i.test(detail)) return null
-    const remedy = describePrOpenFailure(res.status, 'github')
-    const base = redactSecrets(`Failed to open PR (HTTP ${res.status}): ${detail.slice(0, 300)}`)
-    throw new HarnessFailure('api', remedy ? `${base}\n${remedy}` : base)
-  }
-  const body = (await res.json()) as { html_url?: string }
-  if (!body.html_url) throw new HarnessFailure('api', 'GitHub did not return a PR url')
-  return body.html_url
-}
-
-/** GitLab API headers for the PAT (the `PRIVATE-TOKEN` auth GitLab uses). */
-function gitlabHeaders(token: string): Record<string, string> {
-  return {
-    'private-token': token,
-    accept: 'application/json',
-    'user-agent': 'cat-factory-executor',
-    'content-type': 'application/json',
-  }
-}
-
-/**
- * Open a GitLab merge request (the analogue of {@link openPullRequest} for GitLab). The REST
- * base + project path are derived from the clone URL's host, so it works for gitlab.com and a
- * self-managed instance alike. `head`→`source_branch`, `base`→`target_branch`. On a duplicate
- * (a resumed run whose branch already has an open MR — GitLab answers 409) the existing MR's
- * web URL is returned instead of failing the run, mirroring the GitHub 422 handling.
- */
-async function openGitLabMergeRequest(
-  opts: OpenPullRequestOptions & { cloneUrl: string },
-): Promise<string> {
-  const apiBase = gitlabApiBaseFromCloneUrl(opts.cloneUrl)
-  const project = gitlabProjectPath(opts.cloneUrl)
-  const res = await withApiRetry(
-    () =>
-      fetch(`${apiBase}/projects/${project}/merge_requests`, {
-        method: 'POST',
-        headers: gitlabHeaders(opts.ghToken),
-        body: JSON.stringify({
-          source_branch: opts.head,
-          target_branch: opts.base,
-          title: opts.pr.title,
-          description: opts.pr.body,
-        }),
-        ...(opts.signal ? { signal: opts.signal } : {}),
-      }),
-    { signal: opts.signal },
-  )
-  if (!res.ok) {
-    const detail = await res.text().catch(() => '')
-    // GitLab returns 409 (sometimes 400) when an open MR already exists for this source
-    // branch; that is success for a resumed run — return the existing MR's url.
-    if (
-      (res.status === 409 || res.status === 400) &&
-      /already exists|open merge request/i.test(detail)
-    ) {
-      const existing = await findOpenMergeRequestUrl(apiBase, project, opts)
-      if (existing) return existing
-    }
-    const remedy = describePrOpenFailure(res.status, 'gitlab')
-    const base = redactSecrets(
-      `Failed to open merge request (HTTP ${res.status}): ${detail.slice(0, 300)}`,
-    )
-    throw new HarnessFailure('api', remedy ? `${base}\n${remedy}` : base)
-  }
-  const body = (await res.json()) as { web_url?: string }
-  if (!body.web_url) throw new HarnessFailure('api', 'GitLab did not return a merge request url')
-  return body.web_url
-}
-
-/** Find the open GitLab MR for `opts.head`→`opts.base`, returning its web_url or undefined. */
-async function findOpenMergeRequestUrl(
-  apiBase: string,
-  project: string,
-  opts: { head: string; base: string; ghToken: string; signal?: AbortSignal },
-): Promise<string | undefined> {
-  // Filter by BOTH branches: a source branch can have open MRs to several targets, so the
-  // source alone could match an MR against a different base than the one we just tried to open.
-  const query = new URLSearchParams({
-    source_branch: opts.head,
-    target_branch: opts.base,
-    state: 'opened',
-  })
-  const res = await fetch(`${apiBase}/projects/${project}/merge_requests?${query}`, {
-    headers: gitlabHeaders(opts.ghToken),
-    ...(opts.signal ? { signal: opts.signal } : {}),
-  })
-  if (!res.ok) return undefined
-  const list = (await res.json().catch(() => [])) as Array<{ web_url?: string }>
-  return Array.isArray(list) && list[0]?.web_url ? list[0].web_url : undefined
-}
-
-/** Find the open PR for `opts.head` on `opts.base`, returning its html_url or undefined. */
-async function findOpenPullRequestUrl(opts: {
-  owner: string
-  name: string
-  ghToken: string
-  head: string
-  base: string
-  apiBase?: string
-  signal?: AbortSignal
-}): Promise<string | undefined> {
-  const apiBase = opts.apiBase ?? 'https://api.github.com'
-  // Encode the ref-derived query params: a branch/owner containing `&` or `#` would
-  // otherwise split the query string or inject an unintended parameter.
-  const query = new URLSearchParams({
-    head: `${opts.owner}:${opts.head}`,
-    base: opts.base,
-    state: 'open',
-  })
-  const path = `${encodeURIComponent(opts.owner)}/${encodeURIComponent(opts.name)}`
-  const res = await fetch(`${apiBase}/repos/${path}/pulls?${query}`, {
-    headers: {
-      authorization: `Bearer ${opts.ghToken}`,
-      accept: 'application/vnd.github+json',
-      'user-agent': 'cat-factory-executor',
-      'x-github-api-version': '2022-11-28',
-    },
-    ...(opts.signal ? { signal: opts.signal } : {}),
-  })
-  if (!res.ok) return undefined
-  const list = (await res.json().catch(() => [])) as Array<{ html_url?: string }>
-  return Array.isArray(list) && list[0]?.html_url ? list[0].html_url : undefined
-}