@alecsibilia/luca 14.0.0-alpha.2 → 14.0.0-alpha.21

package/engine/tracker/gh-retry.ts

@@ -0,0 +1,217 @@
+/**
+ * Retries for `gh` calls (#569). A call that fails in a way likely to pass
+ * on a second try (a timeout, a cut-off JSON reply, a 5xx, a secondary rate
+ * limit, a dropped connection or a TLS error) is tried again, three tries in
+ * all, after waits of 2 s then 8 s. Any other failure throws at once. Each
+ * retry is logged to the run log.
+ */
+import { TrackerError } from './tracker'
+
+import {
+    durationText,
+    runCommand,
+    type CommandResult,
+} from '../shell/run-command'
+
+/**
+ * Runs `gh` with `args` and collects its output. Never throws on a non-zero
+ * exit. The GitHub tracker makes every call through one, so tests can fake
+ * `gh`.
+ */
+export type GhRunner = (args: string[]) => Promise<
+    Pick<CommandResult, 'exit_code' | 'stdout' | 'stderr'> & {
+        timed_out?: boolean
+    }
+>
+
+/** A `gh` call that runs longer is stopped, and counts as transient. */
+export const GH_TIMEOUT_MS = 60_000
+
+/** The waits before the second and third tries. */
+export const GH_RETRY_WAITS_MS = [2_000, 8_000]
+
+/** How waits and retry lines happen; tests fake both. */
+export type GhRetryDeps = {
+    sleep: (ms: number) => Promise<void>
+    log: (line: string) => void
+}
+
+export const realRetryDeps: GhRetryDeps = {
+    sleep: (ms) => Bun.sleep(ms),
+    log: (line) => {
+        console.log(line)
+    },
+}
+
+/** The real `gh`, on the `PATH`, run in `cwd` (default: the engine's). */
+export const realGh =
+    ({ cwd }: { cwd?: string } = {}): GhRunner =>
+    (args) =>
+        runCommand({
+            cmd: ['gh', ...args],
+            cwd: cwd ?? process.cwd(),
+            timeout_ms: GH_TIMEOUT_MS,
+        })
+
+/** A failed `gh` call likely to pass on a second try. */
+export class TransientGhError extends TrackerError {}
+
+/** What `gh` says when a call fails in a way likely to pass next time. */
+const TRANSIENT_STDERR = [
+    /unexpected end of JSON input/i,
+    /unexpected EOF/i,
+    /HTTP 5\d\d/,
+    /secondary rate limit/i,
+    /connection (reset|refused|timed out)/i,
+    /ECONNRESET|ECONNREFUSED|ETIMEDOUT/,
+    /i\/o timeout|deadline exceeded|timeout exceeded/i,
+    /\bTLS\b/i,
+]
+
+const NOT_FOUND = /HTTP 404\b/
+
+/** A failed `gh` call as a `TrackerError`, in `gh`'s own words. */
+const ghError = ({
+    args,
+    result,
+}: {
+    args: string[]
+    result: Awaited<ReturnType<GhRunner>>
+}): TrackerError => {
+    const what = `\`gh ${args.slice(0, 2).join(' ')}\``
+    if (result.timed_out === true) {
+        return new TransientGhError(
+            `${what} failed (timed out after ${durationText(GH_TIMEOUT_MS)})`
+        )
+    }
+    const message = `${what} failed (exit code ${result.exit_code}): ${result.stderr.trim()}`
+    return TRANSIENT_STDERR.some((pattern) => pattern.test(result.stderr))
+        ? new TransientGhError(message)
+        : new TrackerError(message)
+}
+
+/**
+ * Runs `attempt` until it succeeds, it fails in a way that isn't transient,
+ * or the tries run out; then throws its last error. Before a retry,
+ * `landed` (for writes) looks for what the earlier try may have left: when
+ * it finds it, that is the answer and nothing is tried again.
+ */
+export const withRetries = async <T>({
+    attempt,
+    landed,
+    sleep,
+    log,
+}: GhRetryDeps & {
+    attempt: () => Promise<T>
+    landed?: () => Promise<T | null>
+}): Promise<T> => {
+    for (let tries = 1; ; tries += 1) {
+        try {
+            return await attempt()
+        } catch (error) {
+            const wait = GH_RETRY_WAITS_MS[tries - 1]
+            if (!(error instanceof TransientGhError) || wait === undefined) {
+                throw error
+            }
+            log(
+                `[tracker] ${error.message}; trying again in ${durationText(wait)} (try ${tries + 1} of ${GH_RETRY_WAITS_MS.length + 1})`
+            )
+            await sleep(wait)
+            const earlier = landed === undefined ? null : await landed()
+            if (earlier !== null) return earlier
+        }
+    }
+}
+
+/**
+ * The `gh` calls the tracker makes, each with retries:
+ * - `ok` returns the output, and throws when the call fails;
+ * - `read` parses the output with `parse`; output that doesn't parse as
+ *   JSON counts as a cut-off reply, so as transient;
+ * - `api` reads `gh api <path>` as JSON, and is `null` only for a 404.
+ */
+export const createGhCalls = ({
+    run,
+    sleep,
+    log,
+}: GhRetryDeps & { run: GhRunner }) => {
+    const once = async ({
+        args,
+        not_found_null,
+    }: {
+        args: string[]
+        not_found_null: boolean
+    }): Promise<string | null> => {
+        const result = await run(args)
+        if (result.exit_code === 0 && result.timed_out !== true) {
+            return result.stdout
+        }
+        if (not_found_null && NOT_FOUND.test(result.stderr)) return null
+        throw ghError({ args, result })
+    }
+
+    const parsed = <T>({
+        args,
+        stdout,
+        parse,
+    }: {
+        args: string[]
+        stdout: string
+        parse: (stdout: string) => T
+    }): T => {
+        try {
+            return parse(stdout)
+        } catch (error) {
+            if (!(error instanceof SyntaxError)) throw error
+            throw new TransientGhError(
+                `\`gh ${args.slice(0, 2).join(' ')}\` failed (a cut-off reply: ${error.message})`
+            )
+        }
+    }
+
+    const read = <T>({
+        args,
+        parse,
+        landed,
+    }: {
+        args: string[]
+        parse: (stdout: string) => T
+        landed?: () => Promise<T | null>
+    }): Promise<T> =>
+        withRetries({
+            attempt: async () =>
+                parsed({
+                    args,
+                    stdout: (await once({ args, not_found_null: false })) ?? '',
+                    parse,
+                }),
+            landed,
+            sleep,
+            log,
+        })
+
+    return {
+        read,
+        ok: ({
+            args,
+            landed,
+        }: {
+            args: string[]
+            landed?: () => Promise<string | null>
+        }): Promise<string> =>
+            read({ args, parse: (stdout) => stdout, landed }),
+        api: ({ path }: { path: string }): Promise<unknown> => {
+            const args = ['api', path]
+            return withRetries({
+                attempt: async () => {
+                    const stdout = await once({ args, not_found_null: true })
+                    return stdout === null
+                        ? null
+                        : parsed({ args, stdout, parse: JSON.parse })
+                },
+                sleep,
+                log,
+            })
+        },
+    }
+}

package/engine/tracker/github-tracker.ts

@@ -1,15 +1,25 @@
-import { $ } from 'bun'
-
 import { z } from 'zod'
 
 import {
+    createGhCalls,
+    realGh,
+    realRetryDeps,
+    TransientGhError,
+    type GhRetryDeps,
+    type GhRunner,
+} from './gh-retry'
+import {
+    labelDefinition,
     OpenedPullRequestSchema,
     TrackerCommentSchema,
     TrackerIssueSchema,
+    type LabelDefinition,
     type Tracker,
     type TrackerIssue,
 } from './tracker'
 
+export type { GhRunner } from './gh-retry'
+
 /** An issue as GitHub's REST API returns it (only the fields we use). */
 const GitHubIssueSchema = z.object({
     number: z.number().int().positive(),
@@ -33,17 +43,18 @@ const GitHubIssueNumbersSchema = z.array(
     z.object({ number: z.number().int().positive() })
 )
 
-/** Pull requests as `gh pr list --json number,url` prints them. */
-const GitHubPullListSchema = z.array(OpenedPullRequestSchema)
+/**
+ * Pull requests as `gh pr list --json number,url,isCrossRepository` prints
+ * them. A fork's PR has `isCrossRepository` set.
+ */
+const GitHubPullListSchema = z.array(
+    OpenedPullRequestSchema.extend({ isCrossRepository: z.boolean() })
+)
 
 type GitHubIssue = z.infer<typeof GitHubIssueSchema>
 
-/** Calls `gh api`; `null` when the call fails (such as a 404). */
-const ghApi = async ({ path }: { path: string }): Promise<unknown> => {
-    const result = await $`gh api ${path}`.quiet().nothrow()
-    if (result.exitCode !== 0) return null
-    return JSON.parse(result.stdout.toString())
-}
+/** Whether `gh label create` failed because the label is there already. */
+const ALREADY_EXISTS = /already exists/i
 
 const parseOrThrow = <T>({
     schema,
@@ -65,14 +76,50 @@ const parseOrThrow = <T>({
 
 /**
  * The real tracker: GitHub issues through the `gh` CLI, which must be
- * installed and logged in. Not exercised by tests; the in-memory tracker
- * stands in for it.
+ * installed and logged in. Engine tests use the in-memory tracker; the
+ * tracker's own tests pass a fake `gh`.
+ *
+ * Adding a label the repo lacks creates it first (#500): when the add
+ * fails, the repo's labels are read, and a missing one is created as
+ * `labelDefinition` describes it, then added again. A failure with the
+ * label there is thrown as it is. The happy path costs one call, and
+ * nothing hangs on the wording of `gh`'s error.
+ *
+ * Every call retries transient failures (see `gh-retry.ts`). A read throws
+ * after its last try; only a 404 reads as none. A write checks whether its
+ * first try landed before trying again: a comment with the same body is
+ * adopted, as is an open PR from the same branch, and a label that already
+ * exists counts as created.
+ *
+ * @param gh - Runs `gh`. Defaults to the real one.
+ * @param sleep - Waits between tries. Defaults to a real wait.
+ * @param log - Writes a retry to the run log. Defaults to stdout.
  *
  * @example
  * const tracker = createGitHubTracker({ repo: 'asibilia/luca-framework' })
  * const spec = await tracker.readSpec({ spec_number: 359 })
  */
-export const createGitHubTracker = ({ repo }: { repo: string }): Tracker => {
+export const createGitHubTracker = ({
+    repo,
+    gh,
+    sleep,
+    log,
+}: {
+    repo: string
+    gh?: GhRunner
+} & Partial<GhRetryDeps>): Tracker => {
+    const calls = createGhCalls({
+        run: gh ?? realGh(),
+        sleep: sleep ?? realRetryDeps.sleep,
+        log: log ?? realRetryDeps.log,
+    })
+
+    /** Runs `gh` and returns its output; throws when it fails. */
+    const ghOk = (args: string[]): Promise<string> => calls.ok({ args })
+
+    /** Reads `gh api <path>` as JSON; `null` only for a 404. */
+    const ghApi = calls.api
+
     const blockedBy = async (number: number): Promise<number[]> => {
         const value = await ghApi({
             path: `repos/${repo}/issues/${number}/dependencies/blocked_by`,
@@ -111,6 +158,133 @@ export const createGitHubTracker = ({ repo }: { repo: string }): Tracker => {
         )
     }
 
+    const listLabels: Tracker['listLabels'] = async () =>
+        (
+            await ghOk([
+                'label',
+                'list',
+                '--repo',
+                repo,
+                '--limit',
+                '1000',
+                '--json',
+                'name',
+                '--jq',
+                '.[].name',
+            ])
+        )
+            .split('\n')
+            .filter((name) => name !== '')
+
+    const createLabelArgs = ({
+        name,
+        color,
+        description,
+    }: LabelDefinition): string[] => [
+        'label',
+        'create',
+        name,
+        '--repo',
+        repo,
+        '--color',
+        color,
+        '--description',
+        description,
+    ]
+
+    const listComments: Tracker['listComments'] = async ({
+        number,
+        since_id,
+    }) =>
+        // One JSON object per line, across every page.
+        (
+            await calls.read({
+                args: [
+                    'api',
+                    '--paginate',
+                    `repos/${repo}/issues/${number}/comments?per_page=100`,
+                    '--jq',
+                    '.[] | {id, author: (.user.login // ""), body: (.body // "")}',
+                ],
+                parse: (listed) =>
+                    listed
+                        .split('\n')
+                        .filter((line) => line.trim() !== '')
+                        .map((line) =>
+                            parseOrThrow({
+                                schema: TrackerCommentSchema,
+                                value: JSON.parse(line),
+                                what: `a comment on #${number}`,
+                            })
+                        ),
+            })
+        ).filter(({ id }) => id > since_id)
+
+    /** Creates a label; one already there counts as created. */
+    const createLabel: Tracker['createLabel'] = async (label) => {
+        try {
+            await ghOk(createLabelArgs(label))
+        } catch (error) {
+            if (error instanceof Error && ALREADY_EXISTS.test(error.message)) {
+                return
+            }
+            throw error
+        }
+    }
+
+    /** Adds or removes one label of an issue; never throws. */
+    const editLabel = ({
+        number,
+        label,
+        change,
+    }: {
+        number: number
+        label: string
+        change: 'add' | 'remove'
+    }): Promise<Error | null> =>
+        ghOk([
+            'issue',
+            'edit',
+            String(number),
+            '--repo',
+            repo,
+            `--${change}-label`,
+            label,
+        ]).then(
+            () => null,
+            (error: Error) => error
+        )
+
+    const findOpenPullRequest: Tracker['findOpenPullRequest'] = async ({
+        head,
+    }) => {
+        // `--head` matches the branch name alone, so a fork's PR from a
+        // branch of the same name is left out: it isn't the run's.
+        const pull = await calls.read({
+            args: [
+                'pr',
+                'list',
+                '--repo',
+                repo,
+                '--head',
+                head,
+                '--state',
+                'open',
+                '--json',
+                'number,url,isCrossRepository',
+            ],
+            parse: (listed) =>
+                parseOrThrow({
+                    schema: GitHubPullListSchema,
+                    value: JSON.parse(listed),
+                    what: `the open pull requests from ${head}`,
+                }).find(({ isCrossRepository }) => !isCrossRepository),
+        })
+        return pull === undefined
+            ? null
+            : { number: pull.number, url: pull.url }
+    }
+
     return {
         readIssue,
         readSpec: async ({ spec_number }) => {
@@ -132,79 +306,100 @@ export const createGitHubTracker = ({ repo }: { repo: string }): Tracker => {
             })
             return Promise.all(issues.map(toTrackerIssue))
         },
-        comment: async ({ number, body }) => {
-            const posted =
-                await $`gh api repos/${repo}/issues/${number}/comments -f body=${body}`.quiet()
-            return parseOrThrow({
-                schema: GitHubCommentPostedSchema,
-                value: JSON.parse(posted.stdout.toString()),
-                what: `the comment posted on #${number}`,
-            })
-        },
-        listComments: async ({ number, since_id }) => {
-            // One JSON object per line, across every page.
-            const listed =
-                await $`gh api --paginate repos/${repo}/issues/${number}/comments?per_page=100 --jq ${'.[] | {id, author: (.user.login // ""), body: (.body // "")}'}`.quiet()
-            return listed.stdout
-                .toString()
-                .split('\n')
-                .filter((line) => line.trim() !== '')
-                .map((line) =>
+        // Before a retry, a comment with the same body (its marker is in
+        // it) is the first try landing: it is adopted, not posted again.
+        comment: ({ number, body }) =>
+            calls.read({
+                args: [
+                    'api',
+                    `repos/${repo}/issues/${number}/comments`,
+                    '-f',
+                    `body=${body}`,
+                ],
+                parse: (posted) =>
                     parseOrThrow({
-                        schema: TrackerCommentSchema,
-                        value: JSON.parse(line),
-                        what: `a comment on #${number}`,
-                    })
-                )
-                .filter(({ id }) => id > since_id)
-        },
+                        schema: GitHubCommentPostedSchema,
+                        value: JSON.parse(posted),
+                        what: `the comment posted on #${number}`,
+                    }),
+                landed: async () => {
+                    const comments = await listComments({ number, since_id: 0 })
+                    const same = comments.find((each) => each.body === body)
+                    return same === undefined ? null : { id: same.id }
+                },
+            }),
+        listComments,
         addLabel: async ({ number, label }) => {
-            await $`gh issue edit ${number} --repo ${repo} --add-label ${label}`.quiet()
+            const failed = await editLabel({ number, label, change: 'add' })
+            if (failed === null) return
+            if ((await listLabels()).includes(label)) throw failed
+            // A failed create still tries the add again: another run may
+            // have just created the label.
+            const created = await createLabel(
+                labelDefinition({ name: label })
+            ).then(
+                () => null,
+                (error: Error) => error
+            )
+            const again = await editLabel({ number, label, change: 'add' })
+            if (again === null) return
+            if (created === null) throw again
+            throw new Error(
+                `${again.message}; creating the label failed too: ${created.message}`
+            )
         },
         removeLabel: async ({ number, label }) => {
-            await $`gh issue edit ${number} --repo ${repo} --remove-label ${label}`
-                .quiet()
-                .nothrow()
+            const failed = await editLabel({ number, label, change: 'remove' })
+            if (failed === null) return
+            // A label the repo lacks can't be on the issue.
+            if ((await listLabels()).includes(label)) throw failed
         },
+        // A read that fails fails the close too, rather than closing blind.
         closeIssue: async ({ number }) => {
             const issue = await readIssue({ number })
             if (issue?.state === 'closed') return
-            await $`gh issue close ${number} --repo ${repo} --reason completed`.quiet()
-        },
-        openPullRequest: async ({ head, base, title, body }) => {
-            const url = (
-                await $`gh pr create --repo ${repo} --head ${head} --base ${base} --title ${title} --body ${body}`.quiet()
-            ).stdout
-                .toString()
-                .trim()
-            const number = Number(/\/pull\/(\d+)/.exec(url)?.[1])
-            return parseOrThrow({
-                schema: OpenedPullRequestSchema,
-                value: { number, url },
-                what: `the pull request from ${head}`,
-            })
-        },
-        findOpenPullRequest: async ({ head }) => {
-            const listed =
-                await $`gh pr list --repo ${repo} --head ${head} --state open --json number,url`.quiet()
-            const [pull] = parseOrThrow({
-                schema: GitHubPullListSchema,
-                value: JSON.parse(listed.stdout.toString()),
-                what: `the open pull requests from ${head}`,
-            })
-            return pull ?? null
-        },
-        listLabels: async () => {
-            const listed =
-                await $`gh label list --repo ${repo} --limit 1000 --json name --jq ${'.[].name'}`.quiet()
-            return listed.stdout
-                .toString()
-                .split('\n')
-                .filter((name) => name !== '')
-        },
-        createLabel: async ({ name, color, description }) => {
-            await $`gh label create ${name} --repo ${repo} --color ${color} --description ${description}`.quiet()
+            await ghOk([
+                'issue',
+                'close',
+                String(number),
+                '--repo',
+                repo,
+                '--reason',
+                'completed',
+            ])
         },
+        // Retried only once no open PR from `head` shows the first try
+        // landed; one that does is adopted.
+        openPullRequest: ({ head, base, title, body }) =>
+            calls.read({
+                args: [
+                    'pr',
+                    'create',
+                    '--repo',
+                    repo,
+                    '--head',
+                    head,
+                    '--base',
+                    base,
+                    '--title',
+                    title,
+                    '--body',
+                    body,
+                ],
+                parse: (created) => {
+                    const url = created.trim()
+                    const number = Number(/\/pull\/(\d+)/.exec(url)?.[1])
+                    return parseOrThrow({
+                        schema: OpenedPullRequestSchema,
+                        value: { number, url },
+                        what: `the pull request from ${head}`,
+                    })
+                },
+                landed: () => findOpenPullRequest({ head }),
+            }),
+        findOpenPullRequest,
+        listLabels,
+        createLabel,
         // Checked on the repo's newest issue; a repo with no issues yet
         // can't show them missing, so both count as there.
         issueLinks: async () => {
@@ -245,23 +440,48 @@ export const githubRepoOf = async ({
 }: {
     repo: string
 }): Promise<string> => {
-    const result =
-        await $`gh repo view --json nameWithOwner --jq .nameWithOwner`
-            .cwd(repo)
-            .quiet()
-            .nothrow()
-    const name = result.stdout.toString().trim()
-    if (result.exitCode !== 0 || name === '') {
-        throw new Error(
-            `Could not find the GitHub repo of ${repo} with gh: ${result.stderr.toString().trim()}`
+    const calls = createGhCalls({
+        run: realGh({ cwd: repo }),
+        ...realRetryDeps,
+    })
+    const name = await calls
+        .ok({
+            args: [
+                'repo',
+                'view',
+                '--json',
+                'nameWithOwner',
+                '--jq',
+                '.nameWithOwner',
+            ],
+        })
+        .then(
+            (stdout) => stdout.trim(),
+            (error: Error) => {
+                throw new Error(
+                    `Could not find the GitHub repo of ${repo} with gh: ${error.message}`
+                )
+            }
         )
+    if (name === '') {
+        throw new Error(`Could not find the GitHub repo of ${repo} with gh`)
     }
     return name
 }
 
-/** The login `gh` is signed in as, or `null` when it isn't. */
+/**
+ * The login `gh` is signed in as, or `null` when it isn't. A transient
+ * failure is retried, and throws after the last try.
+ */
 export const ghLogin = async (): Promise<string | null> => {
-    const result = await $`gh api user --jq .login`.quiet().nothrow()
-    const login = result.stdout.toString().trim()
-    return result.exitCode === 0 && login !== '' ? login : null
+    const calls = createGhCalls({ run: realGh(), ...realRetryDeps })
+    try {
+        const login = (
+            await calls.ok({ args: ['api', 'user', '--jq', '.login'] })
+        ).trim()
+        return login === '' ? null : login
+    } catch (error) {
+        if (error instanceof TransientGhError) throw error
+        return null
+    }
 }