@kitman-sh/software-engineering 0.0.0-stage → 0.10.54

package/src/shepherd.ts

@@ -0,0 +1,530 @@
+import {
+  agentWork,
+  approveChangeRequest,
+  ask,
+  call,
+  type CalledLoop,
+  comment,
+  Continuing,
+  defineLoop,
+  mergeChangeRequest,
+  pipelineMoved,
+  pipelineStatus,
+  reportPipeline,
+  rerunPipeline,
+  reviewThreads,
+  syncBase,
+  verify,
+  waitFor,
+  writeBack,
+  isBlocking,
+  type BaseSync,
+  type MergedFinding,
+  type PipelineStatus,
+  type ReviewThreads
+} from '@kitman-sh/sdk/loop'
+import * as Effect from 'effect/Effect'
+import * as Schema from 'effect/Schema'
+import review, { ReviewInput, ReviewOutput } from './review'
+import { ShepherdOutput } from './shepherd-output'
+
+/** Review, by its schemas: Build calls this Loop and Review calls Build, so Review's definition may not be ready yet. */
+const reviewLoop = () => ({ id: 'review', input: ReviewInput, output: ReviewOutput })
+
+/** The model time one fix or round of replies may take before a person looks. */
+const stepBudget = '45 minutes'
+
+const Input = Schema.Struct({
+  /** The open ChangeRequest to see through to merge; the Run works on its branch. */
+  changeRequestId: Schema.String,
+  taskId: Schema.optional(Schema.String),
+  /** Merges once it is approved, green and mergeable, without asking. */
+  merge: Schema.optional(Schema.Boolean),
+  /** The target branch, for merging it in on a conflict, when Kitman does not say the ChangeRequest's own. */
+  branch: Schema.optional(Schema.String),
+  /** The ChangeRequest's branch and target, as Kitman puts the Run on it. */
+  continuing: Schema.optional(Continuing),
+  /** Rounds of fixing a failed pipeline before it stops as needs you; 3 when left out. */
+  maxFixRounds: Schema.optional(Schema.Int),
+  /** Times it looks at the ChangeRequest before asking whether to go on; 20 when left out. */
+  maxCycles: Schema.optional(Schema.Int)
+})
+
+export { ShepherdOutput }
+
+const Output = ShepherdOutput
+
+type Output = typeof Output.Type
+
+const CiFix = Schema.Struct({
+  /** fixed: a commit fixes the branch's own failure; pre_existing: it fails on the target branch too; flaky: it passed on its rerun. */
+  verdict: Schema.Literals(['fixed', 'pre_existing', 'flaky']),
+  summary: Schema.String
+})
+
+const Replies = Schema.Struct({
+  replies: Schema.Array(
+    Schema.Struct({
+      threadId: Schema.String,
+      /** What changed, why not, or the answer to a question; posted on the thread. */
+      reply: Schema.String,
+      /** Whether a commit changed the code for it. */
+      changed: Schema.Boolean
+    })
+  )
+})
+
+const failedJobs = (pipeline: PipelineStatus) =>
+  pipeline.jobs.flatMap(job =>
+    job.status === 'failed'
+      ? [`### ${job.name}${job.url === undefined ? '' : ` (${job.url})`}\n\n${job.logTail ?? 'No log was served.'}`]
+      : []
+  )
+
+const ciFixPrompt = (pipeline: PipelineStatus, target: string) =>
+  [
+    `The pipeline at ${pipeline.head} failed. These jobs failed, with the end of each log:`,
+    ...failedJobs(pipeline),
+    'Run the failing job once more here to rule out a flake; if it passes, change nothing and answer flaky.',
+    `Then run it on the target branch (a worktree of \`origin/${target}\`, which Kitman fetched just before this Step; do not fetch). If it fails there too, the branch did not cause it: change nothing and answer pre_existing.`,
+    "Otherwise it is this branch's own failure: write or fix a test that shows it first, then fix it, and answer fixed. Kitman pushes what you leave."
+  ].join('\n\n')
+
+const threadLine = (thread: ReviewThreads['threads'][number]) =>
+  [
+    `### Thread ${thread.id}${thread.path === undefined ? '' : ` on ${thread.path}${thread.line === undefined ? '' : `:${thread.line}`}`}`,
+    ...thread.comments.map(entry => `${entry.author ?? 'someone'}: ${entry.body}`)
+  ].join('\n')
+
+const ReviewFix = Schema.Struct({ summary: Schema.String })
+
+const reviewFixPrompt = (findings: readonly MergedFinding[]) =>
+  [
+    'A review of the files the catch-up merge changed found these. Fix each blocker and major finding, with a test that shows it where one can. Kitman pushes what you leave.',
+    ...findings.map(finding => `- ${finding.id} (${finding.severity}): ${finding.detail}`)
+  ].join('\n')
+
+type CatchUp = NonNullable<Extract<BaseSync, { readonly status: 'merged' }>['catchUp']>
+
+/**
+ * What a catch-up merge leaves to approve: nothing new, so the approval carries; only the files it settled or changed
+ * after the merge, to review; or a change to the branch's own diff elsewhere, which only a fresh approval covers.
+ */
+const catchUpVerdict = (
+  catchUp: CatchUp
+):
+  | { readonly kind: 'carry' }
+  | { readonly kind: 'review'; readonly files: readonly string[] }
+  | { readonly kind: 'changed'; readonly why: string } => {
+  if (catchUp.truncated) return { kind: 'changed', why: 'it changed more files than the Runner compares' }
+  const files = [...new Set([...catchUp.conflicted, ...catchUp.rewritten])]
+  const elsewhere = catchUp.changed.filter(file => !files.includes(file))
+
+  if (elsewhere.length > 0) return { kind: 'changed', why: `the branch's own diff changed in ${elsewhere.join(', ')}` }
+
+  return files.length === 0 ? { kind: 'carry' } : { kind: 'review', files }
+}
+
+const repliesPrompt = (threads: ReviewThreads['threads']) =>
+  [
+    'Reviewers left these comments on the ChangeRequest. For each thread, apply what you agree with and give the reply to post on it: what you changed, or why you did not. A comment that asks a question gets an answer and no change.',
+    "Never mark a thread resolved: whether a disputed point is settled is the reviewer's call.",
+    ...threads.map(threadLine)
+  ].join('\n\n')
+
+/** A thread's key changes when someone adds to it, so a new comment on an answered thread is answered again. */
+const threadKey = (thread: ReviewThreads['threads'][number]) => `${thread.id}:${thread.comments.length}`
+
+/**
+ * Takes an open ChangeRequest to merge: it waits on its Events with no model time between them, fixes a pipeline the
+ * branch broke (after ruling out a flake and the target branch), answers each new review thread on the thread, merges
+ * the target branch in on a conflict, and merges once approved, green and mergeable when told to or when a person
+ * says so. It stops as needs you, with why, instead of spinning.
+ */
+export default defineLoop({
+  id: 'shepherd',
+  version: '1.1.0',
+  title: 'Shepherd',
+  input: Input,
+  output: Output,
+  agents: { implementer: 'implementer' },
+  checkout: 'write',
+  capabilities: [
+    'agent_work',
+    'read_change_request',
+    'post_comment',
+    'merge_change_request',
+    'approve_change_request',
+    'call',
+    'write_back',
+    'ask',
+    'verify'
+  ],
+  calls: (): readonly CalledLoop[] => [review],
+  run: input => {
+    const changeRequestId = input.changeRequestId
+    // The ChangeRequest's own target: an offer's `branch` may be the offering Run's branch, which is this one.
+    const target = input.continuing?.targetBranch ?? input.branch ?? 'HEAD'
+    const answered = new Set<string>()
+    const reranHeads = new Set<string>()
+    const ranOnRunner = new Set<string>()
+    let fixRounds = 0
+    let replies = 0
+    let stalledAt: string | undefined
+    let stalls = 0
+
+    /** The files a catch-up merge settled or changed, still to review at the head it pushed or a later one. */
+    let unreviewed:
+      | {
+          readonly files: readonly string[]
+          readonly synced: string
+          /** The base sync whose merged-from head's approval covers the rest of the change. */
+          readonly sync: string
+          readonly earlier: readonly MergedFinding[]
+        }
+      | undefined
+
+    /** Why an approval of the head is still needed after a catch-up merge, for a person who is left to give it. */
+    let notCarried: string | undefined
+
+    const done = (status: Output['status'], reason: string): Output => ({ status, reason, fixRounds, replies })
+
+    /** Runs the repository's pipeline on a Runner at `head` and reports it; the reason when it could not run. */
+    const runOnRunner = (step: 'run' | 'rerun', cycle: number, head: string, checks: readonly string[]) =>
+      Effect.gen(function* () {
+        const verification = `pipeline-${step}:${cycle}`
+
+        const failed = yield* verify(verification, { revision: head, checks }).pipe(
+          Effect.as(undefined),
+          Effect.catchTag('StepFailed', error => Effect.succeed(error.reason))
+        )
+
+        if (failed === undefined)
+          yield* reportPipeline(`pipeline-${step === 'run' ? 'report' : 'rereport'}:${cycle}`, {
+            changeRequestId,
+            verification
+          })
+
+        return failed
+      })
+
+    return Effect.gen(function* () {
+      for (let cycle = 1; ; cycle++) {
+        if (cycle > (input.maxCycles ?? 20)) {
+          const goOn = yield* ask(`continue:${cycle}`, {
+            prompt: `Shepherd has looked at this ChangeRequest ${cycle - 1} times without landing it. Keep going?`,
+            options: ['Keep going', 'Stop'],
+            recommended: 'Stop',
+            recommendationReason: `${cycle - 1} looks have not landed it, so something needs a person rather than another look.`,
+            default: 'Stop',
+            timeout: '1 hour'
+          })
+
+          if (goOn === 'Stop') return done('needs_you', `Stopped after ${cycle - 1} looks without landing it`)
+        }
+
+        const pipeline = yield* pipelineStatus(`pipeline:${cycle}`, { changeRequestId })
+
+        // The forge runs no pipeline for this repository: its pipeline runs on a Runner and is reported to Kitman.
+        if (pipeline.status === 'none' && pipeline.runnerChecks !== undefined) {
+          if (ranOnRunner.has(pipeline.head))
+            return done('needs_you', `The pipeline ran on the Runner at ${pipeline.head}, but no result was recorded`)
+          ranOnRunner.add(pipeline.head)
+
+          const failed = yield* runOnRunner('run', cycle, pipeline.head, pipeline.runnerChecks)
+
+          if (failed !== undefined) return done('needs_you', `The pipeline could not run on the Runner: ${failed}`)
+
+          continue
+        }
+
+        if (pipeline.status === 'none' || pipeline.status === 'running') {
+          yield* pipelineMoved(`wait:${cycle}`, changeRequestId, pipeline)
+
+          continue
+        }
+
+        if (pipeline.status === 'failed' || pipeline.status === 'canceled') {
+          if (stalledAt === pipeline.head) stalls++
+          else stalls = 0
+
+          if (stalls >= 2)
+            return done('needs_you', `Two fixes pushed nothing new; the pipeline at ${pipeline.head} still fails`)
+
+          if (fixRounds >= (input.maxFixRounds ?? 3))
+            return done('needs_you', `The pipeline still fails after ${fixRounds} fix rounds`)
+          fixRounds++
+          stalledAt = pipeline.head
+
+          const fixed = yield* agentWork(`ci-fix:${cycle}`, {
+            agent: 'implementer',
+            prompt: ciFixPrompt(pipeline, target),
+            output: CiFix,
+            timeout: stepBudget
+          })
+
+          if (fixed.verdict === 'pre_existing')
+            return done('needs_you', `The failure is pre-existing on the target branch: ${fixed.summary}`)
+
+          if (fixed.verdict === 'flaky') {
+            if (pipeline.source === 'reported') {
+              // Only a pipeline that runs on the Runner can be run again; nothing on the forge can run a report.
+              if (pipeline.runnerChecks === undefined)
+                return done('needs_you', `The failing job passed when run again, outside the forge: ${fixed.summary}`)
+
+              if (reranHeads.has(pipeline.head))
+                return done('needs_you', `The failing job failed again when run again on the Runner: ${fixed.summary}`)
+              reranHeads.add(pipeline.head)
+
+              const failed = yield* runOnRunner('rerun', cycle, pipeline.head, pipeline.runnerChecks)
+
+              if (failed !== undefined) return done('needs_you', `The pipeline could not run on the Runner: ${failed}`)
+
+              continue
+            }
+
+            if (reranHeads.has(pipeline.head))
+              return done('needs_you', `The failing job failed again when run again on the forge: ${fixed.summary}`)
+            reranHeads.add(pipeline.head)
+            yield* rerunPipeline(`rerun:${cycle}`, { changeRequestId })
+            // Until the forge reports the rerun, a read still shows the failure it reran.
+            yield* pipelineMoved(`rerun-wait:${cycle}`, changeRequestId, pipeline).pipe(
+              Effect.catchTag('WaitTimedOut', () => Effect.void)
+            )
+          }
+
+          continue
+        }
+
+        const threads = (yield* reviewThreads(`threads:${cycle}`, { changeRequestId })).threads.filter(
+          thread => !answered.has(threadKey(thread))
+        )
+
+        if (threads.length > 0) {
+          const answers = yield* agentWork(`comments:${cycle}`, {
+            agent: 'implementer',
+            prompt: repliesPrompt(threads),
+            output: Replies,
+            timeout: stepBudget
+          })
+
+          for (const answer of answers.replies) {
+            yield* comment(`reply:${cycle}:${answer.threadId}`, {
+              changeRequestId,
+              threadId: answer.threadId,
+              body: answer.reply
+            })
+            replies++
+          }
+
+          for (const thread of threads) answered.add(`${thread.id}:${thread.comments.length + 1}`)
+
+          if (answers.replies.some(answer => answer.changed)) continue
+        }
+
+        // A review comment posted since this look began goes round again, to be read and answered.
+        const commented = {
+          event: 'change_request.comment',
+          filter: { changeRequestId },
+          ...(pipeline.readAt === undefined ? undefined : { since: pipeline.readAt })
+        }
+
+        const mergeable = yield* waitFor(`mergeable:${cycle}`, {
+          event: 'change_request.mergeable',
+          filter: { changeRequestId },
+          timeout: '1 hour',
+          or: [commented]
+        })
+
+        if (mergeable.type === commented.event) continue
+
+        if (mergeable.payload.mergeable === false) {
+          const synced = yield* syncBase(`sync:${cycle}`, { agent: 'implementer', target })
+
+          if (synced.status === 'conflict')
+            return done('needs_you', `Merging ${target} in conflicts on ${synced.files.join(', ')}: ${synced.reason}`)
+
+          if (synced.status === 'merged') {
+            const caught = synced.catchUp
+
+            const verdict =
+              caught === undefined
+                ? ({ kind: 'changed', why: 'its Runner did not report what it did to the branch' } as const)
+                : catchUpVerdict(caught)
+
+            if (verdict.kind === 'changed') {
+              unreviewed = undefined
+              notCarried = `merging ${target} in changed it: ${verdict.why}`
+            } else if (caught !== undefined && unreviewed !== undefined) {
+              const files = [...new Set([...unreviewed.files, ...(verdict.kind === 'review' ? verdict.files : [])])]
+
+              // With no fix since the files' merge, the review need not cover this merge's target commits; otherwise
+              // `since` keeps covering the fixes, and this merge with them.
+              unreviewed =
+                caught.from === unreviewed.synced
+                  ? { ...unreviewed, files, synced: caught.head }
+                  : { ...unreviewed, files }
+            } else if (caught !== undefined && verdict.kind === 'review') {
+              notCarried = undefined
+              unreviewed = { files: verdict.files, synced: caught.head, sync: `sync:${cycle}`, earlier: [] }
+            } else if (caught !== undefined) {
+              notCarried = undefined
+
+              const carried = yield* approveChangeRequest(`carry:${cycle}`, {
+                changeRequestId,
+                head: caught.head,
+                summary: `Merging ${target} in left the branch's own diff as it was approved.`,
+                carries: `sync:${cycle}`
+              }).pipe(
+                Effect.catchTag('StepFailed', error =>
+                  Effect.succeed({ approved: false as const, reason: error.reason })
+                )
+              )
+
+              if (!carried.approved) notCarried = `the approval was not carried: ${carried.reason}`
+            }
+          }
+
+          continue
+        }
+
+        let reviewedHere = false
+
+        if (unreviewed !== undefined) {
+          const files = unreviewed
+          const head = pipeline.head
+
+          const reviewed = yield* call(`review:${cycle}`, reviewLoop(), {
+            base: `origin/${target}`,
+            head,
+            files: files.files,
+            ...(head === files.synced ? undefined : { since: files.synced }),
+            ...(files.earlier.length === 0 ? undefined : { earlier: files.earlier })
+          })
+
+          if (reviewed.blocking) {
+            const blocking = reviewed.findings.filter(isBlocking)
+
+            if (fixRounds >= (input.maxFixRounds ?? 3))
+              return done(
+                'needs_you',
+                `The review of the catch-up merge's files still finds ${blocking.map(finding => finding.id).join(', ')} after ${fixRounds} fix rounds`
+              )
+            fixRounds++
+
+            yield* agentWork(`review-fix:${cycle}`, {
+              agent: 'implementer',
+              prompt: reviewFixPrompt(blocking),
+              output: ReviewFix,
+              timeout: stepBudget
+            })
+            unreviewed = { ...files, earlier: reviewed.findings }
+
+            continue
+          }
+
+          const approval = yield* approveChangeRequest(`approve:${cycle}`, {
+            changeRequestId,
+            head,
+            summary: `A review of the files merging ${target} in changed (${files.files.join(', ')}) passed with nothing blocking.`,
+            review: `review:${cycle}`,
+            carries: files.sync
+          }).pipe(
+            Effect.catchTag('StepFailed', error => Effect.succeed({ approved: false as const, reason: error.reason }))
+          )
+
+          unreviewed = undefined
+
+          if (approval.approved) reviewedHere = true
+          else
+            notCarried = `merging ${target} in, whose files' review passed but did not approve it: ${approval.reason}`
+        }
+
+        // A newer head's finished pipeline goes round again, to gate that head and wait for its own approval. A
+        // reported pipeline has no forge run to finish on a push, so an approval of a newer head goes round instead.
+        const moved =
+          pipeline.readAt === undefined
+            ? []
+            : [
+                { event: 'change_request.pipeline', filter: { changeRequestId }, since: pipeline.readAt },
+                ...(pipeline.source === 'reported'
+                  ? [
+                      {
+                        event: 'change_request.approval',
+                        filter: { changeRequestId, approved: true },
+                        since: pipeline.readAt
+                      }
+                    ]
+                  : [])
+              ]
+
+        const approved = reviewedHere
+          ? undefined
+          : yield* waitFor(`approved:${cycle}`, {
+              event: 'change_request.approval',
+              // An approval of an earlier commit does not approve the one whose pipeline passed.
+              filter: { changeRequestId, approved: true, head: pipeline.head },
+              timeout: '7 days',
+              or: [commented, ...moved]
+            })
+
+        if (approved !== undefined && approved.type !== 'change_request.approval') continue
+
+        // The approval of a newer head woke a wait on a reported pipeline: that head needs its own pipeline first.
+        if (pipeline.source === 'reported' && approved !== undefined && approved.payload.head !== pipeline.head)
+          continue
+
+        const merge =
+          input.merge === true
+            ? 'Merge'
+            : yield* ask(`merge:${cycle}`, {
+                prompt: 'The ChangeRequest is approved, green and mergeable. Merge now?',
+                options: [
+                  { value: 'Merge', description: 'Merges the head whose pipeline passed.' },
+                  { value: 'Wait', description: 'Leaves the merge to a person.' }
+                ],
+                recommended: 'Merge',
+                recommendationReason: 'It is approved, green and mergeable, which is everything a merge waits for.',
+                default: 'Wait',
+                timeout: '1 hour'
+              })
+
+        if (merge === 'Wait') return done('ready', 'Approved, green and mergeable; left for a person to merge')
+
+        // Only the head whose pipeline passed is merged: one pushed while waiting goes round again.
+        const merged = yield* mergeChangeRequest(`merge-change-request:${cycle}`, {
+          changeRequestId,
+          head: pipeline.head
+        })
+
+        if (merged.status !== 'merged') continue
+        yield* writeBack('write-back', { milestone: 'merged' })
+
+        return done('merged', `Merged at ${merged.revision}`)
+      }
+    }).pipe(
+      // Kitman fails a Step that runs past its time budget with this reason.
+      Effect.catchIf(
+        error => error._tag === 'StepFailed' && error.reason.includes('time budget'),
+        error => Effect.succeed(done('needs_you', `Step "${error.step}" ran past its ${stepBudget} budget`))
+      ),
+      Effect.catchTags({
+        WaitTimedOut: timedOut =>
+          Effect.succeed(
+            done(
+              'needs_you',
+              timedOut.event === 'change_request.approval' && notCarried !== undefined
+                ? `Nobody approved the head after ${notCarried}`
+                : `Nothing happened on the ChangeRequest while waiting for ${timedOut.event}`
+            )
+          ),
+        ChangeRequestClosed: ended =>
+          Effect.succeed(
+            ended.merged
+              ? done('merged', 'Someone else merged the ChangeRequest')
+              : done('needs_you', 'The ChangeRequest was closed')
+          )
+      })
+    )
+  }
+})