@alecsibilia/luca 14.0.0-alpha.2 → 14.0.0-alpha.21

package/engine/core/build-context.ts

@@ -0,0 +1,135 @@
+/**
+ * What a build step's executor works with (`BuildContext`, `buildContext`),
+ * and the small lookups every step shares: a journaled value, a ticket's
+ * worktree, a report file, and the test files in a worktree.
+ */
+import { mkdir } from 'node:fs/promises'
+import { basename, dirname, join } from 'node:path'
+
+import type { AgentLauncher } from '../agents/agent-launcher'
+import type { EngineConfig } from '../config/engine-config'
+import { testFilesAmong } from '../gates/test-runner'
+import type { GitAdapter } from '../git/git-adapter'
+import type { Journal } from '../journal/journal'
+import {
+    replayRun,
+    type ReplayedWorktree,
+    type RunState,
+} from '../journal/replay'
+import type { CommentStep } from '../tracker/post-comment-once'
+import type { Tracker } from '../tracker/tracker'
+
+/** What the engine needs, beyond the journal and tracker, to build tickets. */
+export type BuildDeps = {
+    git: GitAdapter
+    launcher: AgentLauncher
+}
+
+/** What a build step's executor works with. */
+export type BuildContext = BuildDeps & {
+    journal: Journal
+    tracker: Tracker
+    state: RunState
+    config: EngineConfig
+    /** The run's folder, next to its journal and outside git. */
+    run_dir: string
+    /**
+     * Which try of its step this is: a redo after a crash adopts what its
+     * first try left behind.
+     */
+    step: CommentStep
+}
+
+/** A journaled value, or a clear error that the journal has none yet. */
+export const need = <T>({
+    value,
+    what,
+}: {
+    value: T | null | undefined
+    what: string
+}): T => {
+    if (value === null || value === undefined) {
+        throw new Error(`The journal has no ${what} yet.`)
+    }
+    return value
+}
+
+export const ticketWorktree = ({
+    state,
+    ticket,
+}: {
+    state: RunState
+    ticket: number
+}): ReplayedWorktree =>
+    need({
+        value: state.tickets[ticket]?.worktree,
+        what: `worktree for #${ticket}`,
+    })
+
+/** A fresh report file outside the worktree, so it is never a leftover. */
+export const reportFile = async ({
+    context,
+    ticket,
+    label,
+}: {
+    context: BuildContext
+    /** `null` for the final review. */
+    ticket: number | null
+    label: string
+}): Promise<string> => {
+    const folder = join(context.run_dir, 'reports')
+    await mkdir(folder, { recursive: true })
+    return join(
+        folder,
+        `${context.state.last_seq + 1}-${ticket ?? 'final'}-${label}.xml`
+    )
+}
+
+export const testFilesIn = async ({
+    context,
+    cwd,
+}: {
+    context: BuildContext
+    cwd: string
+}): Promise<string[]> =>
+    testFilesAmong({
+        files: await context.git.listFiles({ cwd }),
+        test_file_patterns: context.config.test_file_patterns,
+    })
+
+/**
+ * What a build step's executor works with, from the journal as it is now.
+ *
+ * @example
+ * const context = buildContext({ journal, tracker, git, launcher })
+ */
+export const buildContext = ({
+    journal,
+    tracker,
+    git,
+    launcher,
+    step,
+}: BuildDeps & {
+    journal: Journal
+    tracker: Tracker
+    /** Left out, a first try, numbered after the journal's last record. */
+    step?: CommentStep
+}): BuildContext => {
+    const records = journal.read()
+    const state = replayRun({ records })
+    const run_dir = dirname(journal.file)
+    return {
+        git,
+        launcher,
+        journal,
+        tracker,
+        state,
+        config: need({ value: state.config, what: 'engine config' }),
+        run_dir,
+        step: step ?? {
+            run_id: basename(run_dir),
+            first_seq: (records.at(-1)?.seq ?? 0) + 1,
+            redo: false,
+        },
+    }
+}

package/engine/core/build-launch.ts

@@ -0,0 +1,259 @@
+/**
+ * A fresh agent's launch in the build: the run notes it gets, whether it may
+ * edit tests, and the sections its prompt adds.
+ */
+
+import {
+    isRefactorTicket,
+    type BuildAction,
+    type StepArgs,
+} from './build-action'
+import {
+    CRASH_SECTION,
+    followUpSection,
+    testsUpdatedMessage,
+} from './fix-loop-text'
+import { handoffSection, reviewFixSection, reviewSections } from './review-text'
+import { retrySection } from './stuck-text'
+
+import {
+    CHANGESETS_LENS_SECTION,
+    CHANGESETS_SECTION,
+    rejoinSection,
+    rolePrompt,
+    testUpdateSection,
+} from '../agents/role-prompts'
+import type { AgentRole } from '../agents/role-results'
+import type { TicketSnapshot } from '../intake/intake-schemas'
+import {
+    handoffRole,
+    type ReplayedRunNote,
+    type TicketProgress,
+} from '../journal/replay'
+
+/**
+ * How many run notes a fresh agent is handed: the newest ones in the run,
+ * whatever ticket or role wrote them.
+ */
+export const MAX_RUN_NOTES = 10
+
+/**
+ * The run notes a fresh agent gets: the `MAX_RUN_NOTES` newest, oldest
+ * first. The same text written twice counts once, where it was last written.
+ *
+ * @example
+ * newestRunNotes({ run_notes: state.run_notes }) // at most 10, oldest first
+ */
+export const newestRunNotes = ({
+    run_notes,
+}: {
+    run_notes: ReplayedRunNote[]
+}): ReplayedRunNote[] => {
+    const seen = new Set<string>()
+    const newest: ReplayedRunNote[] = []
+    for (const entry of [...run_notes].reverse()) {
+        if (newest.length === MAX_RUN_NOTES) break
+        if (seen.has(entry.note)) continue
+        seen.add(entry.note)
+        newest.push(entry)
+    }
+    return newest.reverse()
+}
+
+/**
+ * Whether an agent of this role may edit test files on this ticket: only
+ * the test-writer, and a refactor ticket's implementer (who may follow
+ * renames into tests). The engine hands it to the launcher's guards and
+ * uses it for its own after-turn check, on launches and follow-ups alike.
+ *
+ * @example
+ * mayEditTests({ role: 'implementer', ticket }) // true only on a refactor ticket
+ */
+export const mayEditTests = ({
+    role,
+    ticket,
+}: {
+    role: AgentRole
+    ticket: TicketSnapshot
+}): boolean =>
+    role === 'test-writer' ||
+    (role === 'implementer' && isRefactorTicket({ ticket }))
+
+/**
+ * The sections a launch adds: the review's (see `reviewPromptSections`),
+ * and, for a test-writer or implementer fixing a ticket on top of the run
+ * branch, the files that clashed. The reviewer's re-review after a rebase
+ * is the review's own. The first agent after a `retry` resumed a stuck
+ * ticket is also told why it got stuck, and the agent taking a turn a
+ * crash cut off is told of the crash.
+ */
+const promptSections = ({
+    role,
+    progress,
+}: {
+    role: AgentRole
+    progress: TicketProgress
+}): string[] => {
+    const sections = buildPromptSections({ role, progress })
+    const { retried, crashed_turn } = progress
+    return [
+        ...sections,
+        ...(retried === null ? [] : [retrySection({ retried })]),
+        ...(crashed_turn === role ? [CRASH_SECTION] : []),
+    ]
+}
+
+const buildPromptSections = ({
+    role,
+    progress,
+}: {
+    role: AgentRole
+    progress: TicketProgress
+}): string[] => {
+    const review = reviewPromptSections({ role, progress })
+    const { rejoin } = progress
+    if (rejoin === null || role === 'ticket-reviewer') return review
+    // A review fix round after the rejoin has its own findings.
+    if (progress.commits.green !== null) return review
+    const section = rejoinSection({ role, rejoin })
+    return [
+        ...review,
+        ...(section === null ? [] : [section]),
+        ...testUpdateSections({ role, progress }),
+    ]
+}
+
+/**
+ * The sections a launch adds while a test update is open (#489): the
+ * test-writer's task, then, once it answered, the implementer's news.
+ */
+const testUpdateSections = ({
+    role,
+    progress,
+}: {
+    role: AgentRole
+    progress: TicketProgress
+}): string[] => {
+    const update = progress.test_update
+    if (update === null) return []
+    if (role === 'test-writer' && update.result === null) {
+        return [testUpdateSection({ since: update, bad_test: update.bad_test })]
+    }
+    if (role === 'implementer' && update.result !== null) {
+        return [
+            testsUpdatedMessage({
+                bad_test: update.bad_test,
+                summary: update.result.summary,
+            }),
+        ]
+    }
+    return []
+}
+
+/**
+ * The sections a launch adds for the ticket review: the reviewer's diff and
+ * gate results (a re-review's new changes and earlier findings), or a
+ * review fixer's findings while a review fix round is open.
+ */
+const reviewPromptSections = ({
+    role,
+    progress,
+}: {
+    role: AgentRole
+    progress: TicketProgress
+}): string[] => {
+    const fix = progress.review_fix
+    if (role === 'ticket-reviewer') {
+        return reviewSections({
+            progress,
+            base_sha: progress.worktree?.base_sha ?? 'HEAD',
+        })
+    }
+    if (fix === null || progress.commits.green === null) return []
+    if (role === 'test-writer' && !fix.tests_answered) {
+        return [reviewFixSection({ fix, kind: 'test' })]
+    }
+    if (role === 'implementer' && !fix.code_answered) {
+        return [reviewFixSection({ fix, kind: 'code' })]
+    }
+    if (role === handoffRole({ fix })) {
+        return [
+            handoffSection({
+                fix,
+                role,
+                heading: `Review fix round ${fix.round}`,
+            }),
+        ]
+    }
+    return []
+}
+
+/**
+ * A fresh agent session, handed the run's newest notes. With `follow_up`,
+ * it stands in for a follow-up whose session is gone (a crash cut it off),
+ * and its prompt carries the follow-up message.
+ */
+export const launch = ({
+    role,
+    snapshot,
+    ticket,
+    progress,
+    run_notes,
+    follow_up,
+}: StepArgs & { role: AgentRole; follow_up?: string }): BuildAction => ({
+    type: 'launch_agent',
+    ticket: ticket.number,
+    role,
+    prompt: rolePrompt({
+        role,
+        spec: snapshot.spec,
+        ticket,
+        refactor: isRefactorTicket({ ticket }),
+        // A test update names the bad test in its own section.
+        bad_test: progress.test_update === null ? progress.bad_test : null,
+        bad_test_by: progress.bad_test_source ?? undefined,
+        sections: [
+            ...promptSections({ role, progress }),
+            ...(follow_up === undefined
+                ? []
+                : [followUpSection({ message: follow_up })]),
+        ],
+        run_notes,
+    }),
+    may_edit_tests: mayEditTests({ role, ticket }),
+})
+
+/**
+ * A fresh test-writer's or implementer's launch (a ticket's or the final
+ * review's fixers), with `section` added to its prompt, such as the repo's
+ * prepare command. Any other step as it is. With `only`, just that role's
+ * launches get it (the feature map is the test-writer's, #583).
+ */
+export const withWriterSection = ({
+    step,
+    section,
+    only,
+}: {
+    step: BuildAction
+    section: string
+    only?: 'test-writer' | 'implementer'
+}): BuildAction => {
+    const writesCode =
+        (step.type === 'launch_agent' || step.type === 'launch_final_fixer') &&
+        (step.role === 'test-writer' || step.role === 'implementer') &&
+        (only === undefined || step.role === only)
+    return writesCode
+        ? { ...step, prompt: `${step.prompt}\n\n${section}` }
+        : step
+}
+
+/**
+ * In a repo with changesets, the writers are told not to write one (#507),
+ * and every reviewer (the ticket-reviewer and each lens) that the engine adds
+ * the run's one at PR time (#507, #545).
+ */
+export const withChangesetsSections = (step: BuildAction): BuildAction =>
+    step.type === 'launch_lens' ||
+    (step.type === 'launch_agent' && step.role === 'ticket-reviewer')
+        ? { ...step, prompt: `${step.prompt}\n\n${CHANGESETS_LENS_SECTION}` }
+        : withWriterSection({ step, section: CHANGESETS_SECTION })

package/engine/core/commit-in.ts

@@ -0,0 +1,282 @@
+/**
+ * The engine's commits: the leftover scan, then a commit of a worktree's
+ * changes (`commitIn`), with agents' changesets dropped first and a crashed
+ * try's commit adopted. Serves the `commit_ticket` step and the final review.
+ */
+import { rm } from 'node:fs/promises'
+import { basename, join } from 'node:path'
+
+import { need, ticketWorktree, type BuildContext } from './build-context'
+import type { BuildAction } from './decide-build'
+import { leaveOutIn, ticketChanges } from './worktree-changes'
+
+import {
+    importStem,
+    isAgentChangeset,
+    leftoverPasses,
+    nameCheckedFiles,
+    newCodeFiles,
+    scanLeftovers,
+} from '../gates/leftover-scan'
+import { testFilesAmong } from '../gates/test-runner'
+import type { FileChange } from '../git/git-adapter'
+import { pathText } from '../git/path-text'
+import { isTestHelper } from '../guards/role-rules'
+import type { CommitStage } from '../journal/journal-record'
+
+/** Whether anything besides the file itself mentions a new code file. */
+const isUsed = async ({
+    context,
+    cwd,
+    path,
+    added_texts,
+}: {
+    context: BuildContext
+    cwd: string
+    path: string
+    added_texts: Record<string, string>
+}): Promise<boolean> => {
+    const stem = importStem(path)
+    const tracked = await context.git.filesMentioning({ cwd, text: stem })
+    if (tracked.some((file) => file !== path)) return true
+    return Object.entries(added_texts).some(
+        ([file, text]) => file !== path && text.includes(stem)
+    )
+}
+
+/**
+ * Whether a file other than itself and the tests names this scratch-named
+ * file or new markdown (#520) by its basename, as a TOC, `package.json`, a
+ * doc index, or a config does: a
+ * tracked file, or one this change adds (a new addon's TOC names its new
+ * modules). A mention in a test doesn't make a file the repo's.
+ */
+const isNamed = async ({
+    context,
+    cwd,
+    path,
+    added_texts,
+}: {
+    context: BuildContext
+    cwd: string
+    path: string
+    added_texts: Record<string, string>
+}): Promise<boolean> => {
+    const name = basename(path)
+    const tracked = await context.git.filesMentioning({ cwd, text: name })
+    const added = Object.entries(added_texts)
+        .filter(([, text]) => text.includes(name))
+        .map(([file]) => file)
+    const files = [...tracked, ...added].filter((file) => file !== path)
+    const tests = testFilesAmong({
+        files,
+        test_file_patterns: context.config.test_file_patterns,
+    })
+    return files.some((file) => !tests.includes(file))
+}
+
+/**
+ * In a repo with changesets, removes the changesets an agent wrote (see
+ * `isAgentChangeset`) from the worktree and journals `changeset_dropped`:
+ * the engine writes the run's one itself. Returns the other changes.
+ */
+export const dropAgentChangesets = async ({
+    context,
+    cwd,
+    ticket,
+    changes,
+}: {
+    context: BuildContext
+    cwd: string
+    ticket: number | null
+    changes: FileChange[]
+}): Promise<FileChange[]> => {
+    if (!context.state.changesets) return changes
+    const paths = changes.filter(isAgentChangeset).map(({ path }) => path)
+    if (paths.length === 0) return changes
+    for (const path of paths) await rm(join(cwd, path), { force: true })
+    context.journal.append({
+        kind: 'changeset_dropped',
+        ticket,
+        role: null,
+        content: { paths },
+    })
+    return changes.filter(({ path }) => !paths.includes(path))
+}
+
+/**
+ * The commit a crash left behind at `cwd`: HEAD carries `message` and no
+ * record names it yet, so the step's first try made it and died before
+ * journaling it. `null` when HEAD is no such commit.
+ */
+const leftCommit = async ({
+    context,
+    cwd,
+    message,
+}: {
+    context: BuildContext
+    cwd: string
+    message: string
+}): Promise<{ sha: string; message: string; files: string[] } | null> => {
+    const last = await context.git.lastCommit({ cwd })
+    if (last.message !== message.trim()) return null
+    const journaled = context.journal
+        .read()
+        .some(
+            (record) =>
+                (record.kind === 'commit_made' &&
+                    record.content.sha === last.sha) ||
+                (record.kind === 'ticket_joined' &&
+                    record.content.ok &&
+                    record.content.shas.includes(last.sha))
+        )
+    return journaled ? null : { sha: last.sha, message, files: last.files }
+}
+
+/**
+ * The leftover scan, then an engine commit of everything in `cwd`, both
+ * journaled. What the prepare command made (`prepare_made`) and untracked
+ * links pointing outside the checkout (see `leaveOutIn`) are neither
+ * scanned nor committed. An agent's changeset is dropped before the scan
+ * (see `dropAgentChangesets`). A hit blocks the commit. A redo after a crash that
+ * already committed (nothing left to commit, and HEAD is an unjournaled
+ * commit with this message) journals that commit instead of making
+ * another; its scan, of a clean worktree, finds nothing, as the first try's
+ * did before it committed.
+ *
+ * Nothing to commit never runs `git commit`, which would fail (#494): the
+ * current commit is journaled with no files instead. That is a `fix` round
+ * whose every finding was a "won't fix", a green stage whose change a
+ * rebase found already on the run branch, or one whose every change came
+ * from prepare. (A red stage with nothing to commit fails its red check
+ * first, so it gets here only if the worktree changed after that check.)
+ */
+export const commitIn = async ({
+    context,
+    cwd,
+    ticket,
+    stage,
+    message,
+    mention_text,
+}: {
+    context: BuildContext
+    cwd: string
+    /** `null` for the final review. */
+    ticket: number | null
+    stage: CommitStage
+    message: string
+    /** The spec and ticket text a new markdown file may be named in. */
+    mention_text: string
+}) => {
+    const leave_out = await leaveOutIn({ context, cwd, ticket })
+    const changes = await dropAgentChangesets({
+        context,
+        cwd,
+        ticket,
+        changes: await ticketChanges({ context, cwd, leave_out }),
+    })
+    const test_files = testFilesAmong({
+        files: changes.map(({ path }) => path),
+        test_file_patterns: context.config.test_file_patterns,
+    })
+    const added = changes.filter(({ change }) => change === 'added')
+    const added_texts: Record<string, string> = {}
+    for (const { path } of added) {
+        added_texts[path] = (await pathText({ path: join(cwd, path) })) ?? ''
+    }
+    const used_code: Record<string, boolean> = {}
+    for (const path of newCodeFiles({ changes, test_files })) {
+        used_code[path] = await isUsed({ context, cwd, path, added_texts })
+    }
+    const named: string[] = []
+    for (const path of nameCheckedFiles({ changes })) {
+        // A test helper (#619) is the tests', so a test that uses it names it.
+        const helper = isTestHelper({ path, config: context.config })
+        if (
+            helper
+                ? await isUsed({ context, cwd, path, added_texts })
+                : await isNamed({ context, cwd, path, added_texts })
+        ) {
+            named.push(path)
+        }
+    }
+    const inputs = {
+        changes,
+        test_files,
+        mention_text,
+        used_code,
+        named,
+        leftover_allow: context.config.leftover_allow ?? [],
+    }
+    const hits = scanLeftovers(inputs)
+    const passed = leftoverPasses(inputs)
+    context.journal.append({
+        kind: 'leftover_scan',
+        ticket,
+        role: null,
+        content: { stage, hits, ...(passed.length > 0 ? { passed } : {}) },
+    })
+    if (hits.length > 0) return
+    const adopted =
+        changes.length === 0 && context.step.redo
+            ? await leftCommit({ context, cwd, message })
+            : null
+    if (adopted !== null) {
+        context.journal.append({
+            kind: 'commit_made',
+            ticket,
+            role: null,
+            content: { stage, ...adopted },
+        })
+        return
+    }
+    if (changes.length === 0) {
+        // Nothing to commit: `git commit` would fail. For a fix round, the
+        // re-review's new changes are then empty.
+        context.journal.append({
+            kind: 'commit_made',
+            ticket,
+            role: null,
+            content: {
+                stage,
+                sha: await context.git.head({ cwd }),
+                message,
+                files: [],
+            },
+        })
+        return
+    }
+    const commit = await context.git.commitAll({ cwd, message, leave_out })
+    context.journal.append({
+        kind: 'commit_made',
+        ticket,
+        role: null,
+        content: { stage, sha: commit.sha, message, files: commit.files },
+    })
+}
+
+export const commitTicket = async ({
+    context,
+    action,
+}: {
+    context: BuildContext
+    action: Extract<BuildAction, { type: 'commit_ticket' }>
+}) => {
+    const { path: cwd } = ticketWorktree({
+        state: context.state,
+        ticket: action.ticket,
+    })
+    const snapshot = need({
+        value: context.state.snapshot,
+        what: 'spec snapshot',
+    })
+    const ticket = snapshot.tickets[action.ticket]
+    await commitIn({
+        context,
+        cwd,
+        ticket: action.ticket,
+        stage: action.stage,
+        message: action.message,
+        mention_text: [snapshot.spec.body, ticket?.body ?? ''].join('\n'),
+    })
+}