@alecsibilia/luca 14.0.0-alpha.2 → 14.0.0-alpha.21

package/engine/core/execute-join.ts

@@ -0,0 +1,276 @@
+/**
+ * The run branch's joins, undone and redone: a join a crash cut off
+ * (`openJoin`), undoing a ticket's join, rebasing a ticket onto the run
+ * branch, and sending its tests back to the test-writer after a rebase.
+ */
+import uniq from 'lodash/uniq'
+
+import { need, ticketWorktree, type BuildContext } from './build-context'
+import type { BuildAction } from './decide-build'
+import { installIn } from './execute-gates'
+import { closeSessions, openSessionsIn } from './session-close'
+
+import {
+    dependenciesChanged,
+    rebaseNeedsInstall,
+} from '../gates/lockfile-install'
+import { testFilesAmong } from '../gates/test-runner'
+import type { JournalRecord } from '../journal/journal-record'
+import type { RunState } from '../journal/replay'
+
+/**
+ * Where the run branch stood before a join of `ticket` a crash cut off: the
+ * latest `join_started` for it with no `ticket_joined` or `ticket_stuck`
+ * for it after, else `null`.
+ *
+ * @example
+ * openJoin({ records: journal.read(), ticket: 11 }) // 'a1b2c3...' after a crash mid-join
+ */
+export const openJoin = ({
+    records,
+    ticket,
+}: {
+    records: JournalRecord[]
+    ticket: number
+}): string | null => {
+    const last = records.findLast(
+        (record) =>
+            record.ticket === ticket &&
+            (record.kind === 'join_started' ||
+                record.kind === 'ticket_joined' ||
+                record.kind === 'ticket_stuck')
+    )
+    return last?.kind === 'join_started' ? last.content.run_branch_sha : null
+}
+
+/**
+ * Puts the run branch back where it stood before a join of `ticket` a crash
+ * cut off, dropping whatever half of its commits the crash left there. A
+ * cut-off step is taken again before other steps start, so nothing else
+ * moved the run branch since. Does nothing when no join was cut off.
+ */
+export const undoCutOffJoin = async ({
+    context,
+    ticket,
+}: {
+    context: BuildContext
+    ticket: number
+}) => {
+    const sha = openJoin({ records: context.journal.read(), ticket })
+    if (sha === null) return
+    const runBranch = need({
+        value: context.state.run_branch,
+        what: 'run branch',
+    })
+    await context.git.resetWorktree({ cwd: runBranch.path, to: sha })
+}
+
+/**
+ * The run branch commits an undo of `ticket`'s join may take (#519): its
+ * own joined commits, and those of any other ticket that joined and has not
+ * pushed (a later join on top goes with it, and replay clears that join
+ * too). A pushed ticket's commits never.
+ */
+const undoableCommits = ({
+    state,
+    ticket,
+}: {
+    state: RunState
+    ticket: number
+}): string[] =>
+    uniq(
+        Object.entries(state.tickets).flatMap(([number, progress]) => {
+            const { joined } = progress
+            if (!joined?.ok) return []
+            const own = Number(number) === ticket
+            return own || progress.pushed === null ? joined.shas : []
+        })
+    )
+
+/**
+ * Undoes a ticket's join on the run branch, back to before `first_sha`, and
+ * returns the commits undone. It never rewinds over other work (#519): it
+ * may only undo the ticket's own join and later joins not pushed yet
+ * (`undoableCommits`). When git refuses, the run branch is left alone, the
+ * ticket is stuck (`undo_refused`) with why, and it returns `null`, so the
+ * caller does nothing more. The undone joins' install is still in the run
+ * branch's `node_modules`, so when their commits changed dependency files
+ * the lockfile's install runs there again.
+ */
+export const undoJoin = async ({
+    context,
+    ticket,
+    first_sha,
+}: {
+    context: BuildContext
+    ticket: number
+    first_sha: string
+}): Promise<string[] | null> => {
+    const { git, state, journal, launcher } = context
+    const runBranch = need({ value: state.run_branch, what: 'run branch' })
+    const joined = state.tickets[ticket]?.joined
+    const tip = await git.head({ cwd: runBranch.path })
+    const undo = await git.undoReplay({
+        cwd: runBranch.path,
+        first_sha,
+        only: undoableCommits({ state, ticket }),
+    })
+    if (!undo.ok) {
+        // A stuck ticket's agents take no more follow-ups.
+        await closeSessions({
+            journal,
+            launcher,
+            sessions: openSessionsIn({ records: journal.read() }).filter(
+                (session) => session.ticket === ticket
+            ),
+        })
+        journal.append({
+            kind: 'ticket_stuck',
+            ticket,
+            role: null,
+            content: {
+                reason: 'undo_refused',
+                detail: `The engine went to undo this ticket's join on the run branch (from ${first_sha.slice(0, 7)}), but that would have rewound the run branch over other work, so the run branch was left alone:\n${undo.error}`,
+            },
+        })
+        return null
+    }
+    const { undone } = undo
+    const undoneFiles =
+        undone.length === 0
+            ? []
+            : await git.filesBetween({
+                  cwd: runBranch.path,
+                  from: `${first_sha}^`,
+                  to: tip,
+              })
+    // A redo whose first try undid the join before a crash finds nothing
+    // left to undo: the commits it undid are the journaled join's.
+    const named =
+        undone.length === 0 && context.step.redo && joined?.ok === true
+            ? joined.shas
+            : undone
+    if (dependenciesChanged({ changed_files: undoneFiles })) {
+        await installIn({
+            journal,
+            cwd: runBranch.path,
+            target: 'run_branch',
+            ticket: null,
+        })
+    }
+    return named
+}
+
+/**
+ * Puts a joined ticket's change back on top of the run branch: undoes the
+ * join first if asked (its gates failed), then resets the ticket's worktree
+ * to the run branch's tip and applies the ticket's whole diff there,
+ * uncommitted, with conflict markers in the files that clash. Journals the
+ * clashed files split into tests and code. A refused undo (#519) leaves the
+ * ticket stuck, and nothing is rebased.
+ */
+export const rebaseTicket = async ({
+    context,
+    action,
+}: {
+    context: BuildContext
+    action: Extract<BuildAction, { type: 'rebase_ticket' }>
+}) => {
+    const { git, state, journal, config } = context
+    const worktree = ticketWorktree({ state, ticket: action.ticket })
+    const runBranch = need({ value: state.run_branch, what: 'run branch' })
+    const progress = state.tickets[action.ticket]
+    // The ticket's latest commit: a review fix round's, or the green one.
+    const head = need({
+        value: progress?.commits.fix ?? progress?.commits.green,
+        what: `green commit for #${action.ticket}`,
+    })
+    const undone =
+        action.undo_first_sha === null
+            ? []
+            : await undoJoin({
+                  context,
+                  ticket: action.ticket,
+                  first_sha: action.undo_first_sha,
+              })
+    // A refused undo left the run branch alone and the ticket stuck.
+    if (undone === null) return
+    const onto = await git.head({ cwd: runBranch.path })
+    const moved_files = await git.filesBetween({
+        cwd: worktree.path,
+        from: worktree.base_sha,
+        to: onto,
+    })
+    const ticket_files = await git.filesBetween({
+        cwd: worktree.path,
+        from: worktree.base_sha,
+        to: head,
+    })
+    const { conflicts } = await git.rebaseWorktree({
+        cwd: worktree.path,
+        from: worktree.base_sha,
+        to: head,
+        onto,
+    })
+    const tests = testFilesAmong({
+        files: conflicts,
+        test_file_patterns: config.test_file_patterns,
+    })
+    journal.append({
+        kind: 'ticket_rebased',
+        ticket: action.ticket,
+        role: null,
+        content: {
+            cause: action.cause,
+            base_sha: onto,
+            tests,
+            code: conflicts.filter((file) => !tests.includes(file)),
+            undone,
+            reinstall: rebaseNeedsInstall({ moved_files, ticket_files }),
+        },
+    })
+}
+
+/**
+ * Sends a ticket's tests back to its test-writer after a rebase (#489):
+ * looks up the tickets whose joined commits the run branch got between the
+ * ticket's old base and its new one, and the files the run branch changed
+ * there, then journals it all as `tests_sent_back`. Reads git only.
+ */
+export const sendTestsBack = async ({
+    context,
+    action,
+}: {
+    context: BuildContext
+    action: Extract<BuildAction, { type: 'send_tests_back' }>
+}) => {
+    const { git, state, journal } = context
+    const { ticket, round, bad_test, from_sha, base_sha } = action
+    const worktree = ticketWorktree({ state, ticket })
+    const commits = await git.commitsBetween({
+        cwd: worktree.path,
+        from: from_sha,
+        to: base_sha,
+    })
+    const files = await git.filesBetween({
+        cwd: worktree.path,
+        from: from_sha,
+        to: base_sha,
+    })
+    const joined = Object.entries(state.tickets).flatMap(
+        ([number, progress]) => {
+            const other = Number(number)
+            const shas = progress.joined?.ok ? progress.joined.shas : []
+            if (other === ticket || !shas.some((sha) => commits.includes(sha)))
+                return []
+            const title = state.snapshot?.tickets[other]?.title ?? ''
+            return [{ ticket: other, title }]
+        }
+    )
+    journal.append({
+        kind: 'tests_sent_back',
+        ticket,
+        role: null,
+        content: { round, bad_test, from_sha, base_sha, joined, files },
+    })
+}

package/engine/core/execute-memory.ts

@@ -1,6 +1,7 @@
 import type { MemoryAction, MemorySaveStep } from './decide-memory'
 import { buildContext, need, runTurn, type BuildDeps } from './execute-build'
 
+import { launchPick } from '../jev/model-pick'
 import type { Journal } from '../journal/journal'
 import type { JournalRecord } from '../journal/journal-record'
 import { replayRun } from '../journal/replay'
@@ -407,6 +408,14 @@ export const executeMemoryAction = async ({
                 value: built.state.spec_number,
                 what: 'spec',
             })
+            const { model, model_reason } = launchPick({
+                records: journal.read(),
+                state: built.state,
+                config: built.config,
+                ticket: null,
+                role: 'learner',
+                step,
+            })
             journal.append({
                 kind: 'agent_started',
                 ticket: null,
@@ -415,6 +424,8 @@ export const executeMemoryAction = async ({
                     role: 'learner',
                     prompt: action.prompt,
                     follow_up_of: null,
+                    model,
+                    model_reason,
                 },
             })
             await runTurn({
@@ -433,6 +444,7 @@ export const executeMemoryAction = async ({
                         config: built.config,
                         // The learner writes nothing and talks to nobody.
                         messaging: null,
+                        model,
                     }),
             })
             return

package/engine/core/execute-pull-request.ts

@@ -0,0 +1,84 @@
+import { join } from 'node:path'
+
+import type { BuildAction } from './decide-build'
+import type { BuildContext } from './execute-build'
+import { openTicketsNotInRun } from './not-in-run'
+import {
+    fitPullRequestBody,
+    PULL_REQUEST_FILE,
+    withNotInRun,
+} from './pull-request-text'
+
+import { postCommentOnce } from '../tracker/post-comment-once'
+import { isTrackerFailure, type OpenedPullRequest } from '../tracker/tracker'
+
+const errorText = (error: unknown): string =>
+    error instanceof Error ? error.message : String(error)
+
+/**
+ * Opens the run's one pull request, then journals `pull_request_opened`
+ * (#508). Its full text (with the spec's open tickets not in the run) is
+ * saved first to `pull-request.md` in the run folder, so it's never lost.
+ * An open PR from the run branch is reused, never opened twice: the one a
+ * try opened before a crash or a failure, or one the owner opened by hand.
+ * The body sent keeps under `PULL_REQUEST_BUDGET`, and what doesn't fit
+ * is posted as PR comments after it (`fitPullRequestBody`), each once. A
+ * failed tracker call (`isTrackerFailure`: finding or opening the PR, such
+ * as a failed `gh pr create`) journals the run stuck (`pull_request_failed`)
+ * with its error, for the owner's `retry`; any other error is a crash.
+ */
+export const openPullRequest = async ({
+    context,
+    action,
+}: {
+    context: BuildContext
+    action: Extract<BuildAction, { type: 'open_pull_request' }>
+}): Promise<void> => {
+    const { journal, tracker, state, run_dir, step } = context
+    const { head, base, title, body } = action
+    // The spec's open tickets not in the run are named in the PR.
+    const full = withNotInRun({
+        body,
+        open: await openTicketsNotInRun({ tracker, state }),
+    })
+    await Bun.write(join(run_dir, PULL_REQUEST_FILE), full)
+    const fitted = fitPullRequestBody({ body: full })
+    let opened: OpenedPullRequest
+    try {
+        opened =
+            (await tracker.findOpenPullRequest({ head })) ??
+            (await tracker.openPullRequest({
+                head,
+                base,
+                title,
+                body: fitted.body,
+            }))
+    } catch (error) {
+        if (!isTrackerFailure(error)) throw error
+        journal.append({
+            kind: 'run_stuck',
+            ticket: null,
+            role: null,
+            content: {
+                reason: 'pull_request_failed',
+                detail: errorText(error),
+            },
+        })
+        return
+    }
+    for (const [n, comment] of fitted.comments.entries()) {
+        await postCommentOnce({
+            tracker,
+            number: opened.number,
+            body: comment,
+            step,
+            n,
+        })
+    }
+    journal.append({
+        kind: 'pull_request_opened',
+        ticket: null,
+        role: null,
+        content: { ...opened, head, base, title, body: full },
+    })
+}

package/engine/core/execute-red-check.ts

@@ -0,0 +1,219 @@
+/**
+ * The `run_red_check` step: the test-writer's tests run in the ticket's
+ * worktree, checked against its baseline, its test map, and the feature
+ * map, and journaled as `red_check`.
+ */
+import { join } from 'node:path'
+
+import uniq from 'lodash/uniq'
+
+import {
+    need,
+    reportFile,
+    testFilesIn,
+    ticketWorktree,
+    type BuildContext,
+} from './build-context'
+import type { BuildAction } from './decide-build'
+import { leaveOutIn, ticketChanges, watchPrepare } from './worktree-changes'
+
+import {
+    bunTestCommands,
+    featureMapPath,
+    hasSlowTests,
+    testTimeoutOf,
+} from '../config/engine-config'
+import { readFeatureMap } from '../gates/feature-map'
+import { prepareCheck } from '../gates/gate-runner'
+import { endedText } from '../gates/gate-schemas'
+import { checkRed, type ChangedTestSource } from '../gates/red-check'
+import {
+    fileTestsCommand,
+    runBunTests,
+    runTests,
+    testFilesAmong,
+    withOwnRun,
+} from '../gates/test-runner'
+import { pathText } from '../git/path-text'
+
+/**
+ * The red check's problem when the test-writer changed no file (what
+ * prepare made aside): the red commit would have nothing to commit.
+ */
+export const NO_RED_CHANGES =
+    'No file changed, so the red commit would be empty: write the tests for the criteria in the worktree (new or changed test files).'
+
+/**
+ * The repo's feature map as the red check needs it (#583), read from the
+ * worktree at `cwd`: its configured path (a file or a folder of pages,
+ * #597), and the test files it lists. `files` is `null` when the map is
+ * missing or can't be read, so the red check can name it rather than crash.
+ * `null` when the config names no map.
+ */
+const featureMapIn = async ({
+    context,
+    cwd,
+}: {
+    context: BuildContext
+    cwd: string
+}): Promise<{ path: string; files: string[] | null } | null> => {
+    const path = featureMapPath({ config: context.config })
+    if (path === null) return null
+    return {
+        path,
+        files: await readFeatureMap({
+            cwd,
+            path,
+            test_file_patterns: context.config.test_file_patterns,
+        }),
+    }
+}
+
+export const runRedCheck = async ({
+    context,
+    action,
+}: {
+    context: BuildContext
+    action: Extract<BuildAction, { type: 'run_red_check' }>
+}) => {
+    const { path, base_sha } = ticketWorktree({
+        state: context.state,
+        ticket: action.ticket,
+    })
+    const baseline = need({
+        value: context.state.tickets[action.ticket]?.baseline,
+        what: `baseline test run for #${action.ticket}`,
+    })
+    const test_files = await testFilesIn({ context, cwd: path })
+    // Before prepare runs, so what it makes now can't count as a change.
+    const leave_out = await leaveOutIn({
+        context,
+        cwd: path,
+        ticket: action.ticket,
+    })
+    const changed = await ticketChanges({ context, cwd: path, leave_out })
+    // Every test file that differs from the ticket's base, not only from
+    // HEAD: after a bad-test bounce, HEAD may still hold an old red commit,
+    // whose test files the fresh test-writer may not touch (#581).
+    const base_changed = testFilesAmong({
+        files: (
+            await context.git.changedSince({ cwd: path, from: base_sha })
+        ).filter((file) => !leave_out.includes(file)),
+        test_file_patterns: context.config.test_file_patterns,
+    })
+    const prepare = await watchPrepare({
+        context,
+        cwd: path,
+        ticket: action.ticket,
+        run: () => prepareCheck({ config: context.config, cwd: path }),
+    })
+    const files = uniq(
+        action.mapping.flatMap(({ tests }) => tests.map(({ file }) => file))
+    )
+    // A lone rerun (#544) runs each test command alone.
+    const exclusive = action.exclusive === true
+    const fast = await runBunTests({
+        cwd: path,
+        commands: bunTestCommands(context),
+        test_files,
+        timeout_ms: testTimeoutOf({ config: context.config }),
+        test_concurrency: context.config.test_concurrency,
+        exclusive,
+        report_file: await reportFile({
+            context,
+            ticket: action.ticket,
+            label: 'red',
+        }),
+    })
+    // With a slow command, the fast tests leave out every slow test file
+    // (#564): the files the test map names or the ticket changed, that the
+    // fast run gave no results for, run on their own, with plain `bun test`.
+    // A run that gives no results at all leaves the fast run as it is (see
+    // `withOwnRun`): its files don't load, as the fast run said already.
+    const own = hasSlowTests({ config: context.config })
+        ? uniq([
+              ...files,
+              ...testFilesAmong({
+                  files: changed.map((change) => change.path),
+                  test_file_patterns: context.config.test_file_patterns,
+              }),
+          ]).filter((file) => fast.files_without_results.includes(file))
+        : []
+    const current =
+        own.length === 0
+            ? fast
+            : withOwnRun({
+                  fast,
+                  own: await runTests({
+                      cwd: path,
+                      command: fileTestsCommand({ files: own }),
+                      test_files: own,
+                      report_file: await reportFile({
+                          context,
+                          ticket: action.ticket,
+                          label: 'red-own',
+                      }),
+                      timeout_ms: testTimeoutOf({ config: context.config }),
+                      test_concurrency: context.config.test_concurrency,
+                      exclusive,
+                  }),
+                  test_files,
+              })
+    const sources: Record<string, string | null> = {}
+    for (const file of files) {
+        sources[file] = await pathText({ path: join(path, file) })
+    }
+    // Each of those test files as it was at the ticket's base and as it is
+    // now, so the red check can find each new test (#581).
+    const changed_test_sources: ChangedTestSource[] = []
+    for (const file of base_changed) {
+        changed_test_sources.push({
+            file,
+            base: await context.git.fileAt({
+                cwd: path,
+                sha: base_sha,
+                path: file,
+            }),
+            current: await pathText({ path: join(path, file) }),
+        })
+    }
+    const checked = checkRed({
+        criteria_ids: action.criteria_ids,
+        mapping: action.mapping,
+        baseline,
+        current,
+        test_files,
+        sources,
+        changed_test_sources,
+        feature_map: await featureMapIn({ context, cwd: path }),
+    })
+    // A failed prepare after the test-writer's turn is one more problem,
+    // for the test-writer's fix loop; so is a turn that changed no file,
+    // which would leave the red commit nothing to commit (#494).
+    const problems = [
+        ...(prepare === null || prepare.ok
+            ? []
+            : [
+                  `The prepare command \`${prepare.command}\` failed (${endedText(prepare)}):\n${prepare.output}`,
+              ]),
+        ...(changed.length > 0 ? [] : [NO_RED_CHANGES]),
+    ]
+    const result =
+        problems.length === 0
+            ? checked
+            : {
+                  ...checked,
+                  ok: false,
+                  problems: [...problems, ...checked.problems],
+              }
+    context.journal.append({
+        kind: 'red_check',
+        ticket: action.ticket,
+        role: null,
+        content: {
+            ...result,
+            tests: current,
+            ...(exclusive ? { exclusive } : {}),
+        },
+    })
+}