@alecsibilia/luca 14.0.0-alpha.2 → 14.0.0-alpha.21

package/engine/testing/setup-repo.ts

@@ -0,0 +1,194 @@
+import { realpathSync } from 'node:fs'
+import { mkdir, mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+import { $ } from 'bun'
+
+import { afterEach, beforeEach, expect } from 'bun:test'
+
+import type { FakeMuninn } from './fake-muninn'
+import { git } from './practice-repo'
+
+import { runSetup } from '../cli/setup'
+
+/**
+ * `luca setup` end to end (seam 4): a throwaway repo with a local bare
+ * `origin`, a fake GitHub (labels, the `gh` login, the repo, and its issue
+ * links), and a fake MuninnDB. Shared by `cli/setup.test.ts` and
+ * `cli/setup-checks.test.ts`: each calls `useSetupRepo()` once, and reads
+ * `root`, `repo`, `origin`, and `logs` as they stand in the running test.
+ */
+
+export const CONFIG_PATH = join('.luca', 'config.json')
+
+/** The keys a new-style engine config may have. */
+export const NEW_CONFIG_KEYS = [
+    'checks',
+    'test_file_patterns',
+    'test_setup_files',
+    'rule_files',
+    'muninn',
+    'run_budget_tokens',
+]
+
+/** A `package.json` shaped like tmnb's: a bun test, a vitest file, and more. */
+export const TMNB_PACKAGE = JSON.stringify(
+    {
+        name: 'tmnb',
+        private: true,
+        scripts: {
+            dev: 'next dev',
+            test: 'bun test',
+            'test:workers': 'vitest run --config vitest.workers.config.ts',
+            'type-check': 'tsc --noEmit',
+            lint: 'eslint .',
+            'syncpack:check': 'syncpack lint',
+        },
+    },
+    null,
+    4
+)
+
+/** An old-Luca config: its own keys, and the vault to keep. */
+export const OLD_LUCA_CONFIG = JSON.stringify(
+    {
+        lucaVersion: '13.1.0-alpha.0',
+        oversight: 'full-auto',
+        preferences: { schemaVersion: 1 },
+        muninn: {
+            vault: 'tmnb',
+            todoBacklog: { vault: 'tmnb', rootId: 'x' },
+        },
+    },
+    null,
+    2
+)
+
+type FakeLabel = { name: string; color: string; description: string }
+
+/** A fake GitHub side for the repo: labels, the login, and issue links. */
+export const fakeGitHub = ({
+    labels = [],
+    login = 'asibilia',
+    repo_name = 'asibilia/tmnb',
+    sub_issues = true,
+    dependencies = true,
+}: {
+    labels?: FakeLabel[]
+    login?: string | null
+    repo_name?: string | null
+    sub_issues?: boolean
+    dependencies?: boolean
+} = {}) => {
+    const store = labels.map((label) => ({ ...label }))
+    const created: string[] = []
+    return {
+        github: {
+            login: async () => login,
+            githubRepo: async () => repo_name,
+            issueLinks: async () => ({ sub_issues, dependencies }),
+            listLabels: async () => store.map(({ name }) => name),
+            createLabel: async ({ name, color, description }: FakeLabel) => {
+                if (store.some((label) => label.name === name)) {
+                    throw new Error(`label ${name} already exists`)
+                }
+                created.push(name)
+                store.push({ name, color, description })
+            },
+        },
+        /** Every label now, as stored. */
+        labels: () => store.map((label) => ({ ...label })),
+        /** The names of the labels setup created, oldest first. */
+        created: () => [...created],
+    }
+}
+
+export let root = ''
+export let repo = ''
+export let origin = ''
+export const logs: string[] = []
+const log = (line: string) => {
+    logs.push(line)
+}
+
+/** Makes fresh throwaway folders before each test, and removes them after. */
+export const useSetupRepo = () => {
+    beforeEach(async () => {
+        root = realpathSync(await mkdtemp(join(tmpdir(), 'setup-repo-')))
+        repo = join(root, 'repo')
+        origin = join(root, 'origin.git')
+        logs.length = 0
+    })
+
+    afterEach(async () => {
+        await rm(root, { recursive: true, force: true })
+    })
+}
+
+/**
+ * A throwaway repo with these files in its first commit and a local bare
+ * `origin`. `main` is pushed unless `push` is false.
+ */
+export const makeRepo = async ({
+    files,
+    push = true,
+}: {
+    files: Record<string, string>
+    push?: boolean
+}) => {
+    await $`git init -q --bare -b main ${origin}`.quiet()
+    await $`git init -q -b main ${repo}`.quiet()
+    const hooks = join(root, 'no-hooks')
+    await mkdir(hooks)
+    await git(repo, 'config', 'user.name', 'Practice')
+    await git(repo, 'config', 'user.email', 'practice@example.com')
+    await git(repo, 'config', 'commit.gpgsign', 'false')
+    await git(repo, 'config', 'core.hooksPath', hooks)
+    await Bun.write(join(repo, 'README.md'), '# Throwaway\n')
+    for (const [path, content] of Object.entries(files)) {
+        await Bun.write(join(repo, path), content)
+    }
+    await git(repo, 'add', '-A')
+    await git(repo, 'commit', '-q', '-m', 'initial')
+    await git(repo, 'remote', 'add', 'origin', origin)
+    if (push) await git(repo, 'push', '-q', 'origin', 'main')
+}
+
+export const setup = async ({
+    github,
+    memory,
+}: {
+    github: ReturnType<typeof fakeGitHub>['github']
+    memory: FakeMuninn
+}) => runSetup({ repo, github, memory, log })
+
+/** Everything the command printed, its end message included. */
+export const printed = (end: { message: string }): string =>
+    [...logs, end.message].join('\n')
+
+export const readConfigText = (): Promise<string> =>
+    Bun.file(join(repo, CONFIG_PATH)).text()
+
+export const checkNamed = (
+    end: Awaited<ReturnType<typeof setup>>,
+    name: string
+): { name: string; status: string; detail: string } => {
+    const found = end.checks.find((check) => check.name === name)
+    if (found === undefined) throw new Error(`No check named ${name}`)
+    return found
+}
+
+/** Checks that a check is a to-do whose fix is printed. */
+export const expectToDo = (
+    end: Awaited<ReturnType<typeof setup>>,
+    name: string
+) => {
+    const check = checkNamed(end, name)
+    expect(check.status).toBe('todo')
+    expect(check.detail.trim()).not.toBe('')
+    expect(printed(end)).toContain(check.detail)
+}
+
+export const commitCount = async (cwd: string): Promise<string> =>
+    (await git(cwd, 'rev-list', '--count', 'main')).trim()

package/engine/testing/step-record-fixtures.ts

@@ -0,0 +1,55 @@
+/**
+ * Journal entry builders for the scheduler's step records: a step started
+ * and a step ended.
+ */
+import type { AgentRole } from '../agents/role-results'
+import type { JournalEntry } from '../journal/journal-record'
+
+/**
+ * The scheduler started a step: `step` is its action type, plus `:<role>`
+ * for an agent's turn. `key` defaults to the ticket's number, or `run`.
+ */
+export const stepStarted = ({
+    ticket,
+    step,
+    key,
+    role,
+    first_seq,
+}: {
+    ticket: number | null
+    step: string
+    key?: string
+    role?: AgentRole
+    /** The step's first try, on a redo. Defaults to `null`. */
+    first_seq?: number
+}): JournalEntry => ({
+    kind: 'step_started',
+    ticket,
+    role: role ?? null,
+    content: {
+        key: key ?? (ticket === null ? 'run' : String(ticket)),
+        step,
+        first_seq: first_seq ?? null,
+    },
+})
+
+/** The scheduler saw a step settle. `key` as in `stepStarted`. */
+export const stepEnded = ({
+    ticket,
+    step,
+    key,
+    role,
+}: {
+    ticket: number | null
+    step: string
+    key?: string
+    role?: AgentRole
+}): JournalEntry => ({
+    kind: 'step_ended',
+    ticket,
+    role: role ?? null,
+    content: {
+        key: key ?? (ticket === null ? 'run' : String(ticket)),
+        step,
+    },
+})

package/engine/testing/ticket-step-fixtures.ts

@@ -0,0 +1,272 @@
+/**
+ * Journal entry builders for the steps along a ticket's spine: the red check,
+ * the leftover scan, commits, gates, resets, the join and its rebase, the
+ * push, a stuck ticket, and `ticketBuilt`, every step of one ticket passing.
+ */
+import { implemented, reviewed, testsWritten } from './agent-turn-fixtures'
+import {
+    baselineTests,
+    emptyTestRun,
+    RUN_BRANCH,
+    ticketWorktreeCreated,
+} from './run-fixtures'
+
+import type { BadTest } from '../agents/role-results'
+import type { TestCase } from '../gates/gate-schemas'
+import type {
+    CommitStage,
+    JournalEntry,
+    StuckReason,
+} from '../journal/journal-record'
+
+export const redCheck = ({
+    ticket,
+    ok,
+}: {
+    ticket: number
+    ok: boolean
+}): JournalEntry => ({
+    kind: 'red_check',
+    ticket,
+    role: null,
+    content: {
+        ok,
+        problems: ok ? [] : ['"sum adds two numbers" passes already'],
+        notes: [],
+        tests: {
+            ...emptyTestRun(),
+            output: ok ? '1 fail' : '(pass) sum adds two numbers',
+        },
+    },
+})
+
+export const leftoverScan = ({
+    ticket,
+    stage,
+    hits,
+}: {
+    /** `null` for the final review. */
+    ticket: number | null
+    stage: CommitStage
+    hits?: { path: string; reason: string }[]
+}): JournalEntry => ({
+    kind: 'leftover_scan',
+    ticket,
+    role: null,
+    content: { stage, hits: hits ?? [] },
+})
+
+export const commitMade = ({
+    ticket,
+    stage,
+    sha,
+    files,
+}: {
+    /** `null` for the final review. */
+    ticket: number | null
+    stage: CommitStage
+    /** Defaults to `<stage>-sha`. */
+    sha?: string
+    files?: string[]
+}): JournalEntry => ({
+    kind: 'commit_made',
+    ticket,
+    role: null,
+    content: {
+        stage,
+        sha: sha ?? `${stage}-sha`,
+        message: stage,
+        files: files ?? [],
+    },
+})
+
+export const gatesRun = ({
+    ticket,
+    target,
+    ok,
+    cases,
+}: {
+    /** `null` for the final review. */
+    ticket: number | null
+    target: 'ticket' | 'run_branch'
+    ok: boolean
+    /** The test cases a ticket's gates journal (#581). Defaults to none. */
+    cases?: TestCase[]
+}): JournalEntry => ({
+    kind: 'gates_run',
+    ticket,
+    role: null,
+    content: {
+        target,
+        ok,
+        ...(cases === undefined ? {} : { cases }),
+        checks: [
+            {
+                name: 'test',
+                command: 'bun test',
+                ok,
+                exit_code: ok ? 0 : 1,
+                output: ok ? '' : '1 fail',
+            },
+        ],
+    },
+})
+
+/**
+ * The engine threw away a ticket worktree's uncommitted changes. With a
+ * `bad_test`, the engine found that test checks nothing (#581).
+ */
+export const worktreeReset = ({
+    ticket,
+    bad_test,
+}: {
+    ticket: number
+    bad_test?: BadTest
+}): JournalEntry => ({
+    kind: 'worktree_reset',
+    ticket,
+    role: null,
+    content: {
+        sha: 'red-sha',
+        ...(bad_test === undefined ? {} : { bad_test }),
+    },
+})
+
+export const joined = ({ ticket }: { ticket: number }): JournalEntry => ({
+    kind: 'ticket_joined',
+    ticket,
+    role: null,
+    content: { ok: true, shas: ['r1', 'g1'] },
+})
+
+/** A join whose cherry-pick clashed with the run branch (git aborted it). */
+export const joinClashed = ({
+    ticket,
+    error,
+}: {
+    ticket: number
+    /** Defaults to a conflict in `src/index.ts`. */
+    error?: string
+}): JournalEntry => ({
+    kind: 'ticket_joined',
+    ticket,
+    role: null,
+    content: { ok: false, error: error ?? 'CONFLICT (content): src/index.ts' },
+})
+
+/** The engine put a ticket's change back on top of the run branch. */
+export const ticketRebased = ({
+    ticket,
+    cause,
+    tests,
+    code,
+    undone,
+}: {
+    ticket: number
+    cause: 'clash' | 'join_gates'
+    /** The test files that clashed. Defaults to none. */
+    tests?: string[]
+    /** The code files that clashed. Defaults to none. */
+    code?: string[]
+    /** The run branch commits undone first. Defaults to none. */
+    undone?: string[]
+}): JournalEntry => ({
+    kind: 'ticket_rebased',
+    ticket,
+    role: null,
+    content: {
+        cause,
+        base_sha: 'onto-sha',
+        tests: tests ?? [],
+        code: code ?? [],
+        undone: undone ?? [],
+    },
+})
+
+/**
+ * After a rebase, the engine sent a ticket's tests back to its test-writer
+ * (#489): #12 "Add product" joined, and `src/index.ts` changed.
+ */
+export const testsSentBack = ({
+    ticket,
+    round,
+    reason,
+}: {
+    ticket: number
+    /** Defaults to 1. */
+    round?: number
+    /** The bad test's reason. Defaults to "Wrong sum." */
+    reason?: string
+}): JournalEntry => ({
+    kind: 'tests_sent_back',
+    ticket,
+    role: null,
+    content: {
+        round: round ?? 1,
+        bad_test: {
+            file: 'src/sum.test.ts',
+            name: 'sum adds two numbers',
+            reason: reason ?? 'Wrong sum.',
+        },
+        from_sha: 'b0',
+        base_sha: 'onto-sha',
+        joined: [{ ticket: 12, title: 'Add product' }],
+        files: ['src/index.ts', 'src/product.ts'],
+    },
+})
+
+export const ticketStuck = ({
+    ticket,
+    reason,
+    detail,
+}: {
+    ticket: number
+    reason: StuckReason
+    /** Defaults to "why". */
+    detail?: string
+}): JournalEntry => ({
+    kind: 'ticket_stuck',
+    ticket,
+    role: null,
+    content: { reason, detail: detail ?? 'why' },
+})
+
+export const pushed = ({
+    ticket,
+    sha,
+}: {
+    /** `null` for the final review's push. */
+    ticket: number | null
+    /** Defaults to `g1`. */
+    sha?: string
+}): JournalEntry => ({
+    kind: 'run_branch_pushed',
+    ticket,
+    role: null,
+    content: { branch: RUN_BRANCH, sha: sha ?? 'g1' },
+})
+
+/** Every step of one ticket, from its worktree to the push, all passing. */
+export const ticketBuilt = ({ ticket }: { ticket: number }): JournalEntry[] => [
+    ticketWorktreeCreated({ ticket }),
+    baselineTests({ ticket }),
+    testsWritten({ ticket }),
+    redCheck({ ticket, ok: true }),
+    leftoverScan({ ticket, stage: 'red' }),
+    commitMade({ ticket, stage: 'red' }),
+    implemented({ ticket }),
+    gatesRun({ ticket, target: 'ticket', ok: true }),
+    leftoverScan({ ticket, stage: 'green' }),
+    commitMade({ ticket, stage: 'green' }),
+    reviewed({ ticket }),
+    joined({ ticket }),
+    gatesRun({ ticket, target: 'run_branch', ok: true }),
+    pushed({ ticket }),
+]
+
+/** One ticket's steps up to and including its approving review. */
+export const ticketApproved = ({
+    ticket,
+}: {
+    ticket: number
+}): JournalEntry[] => ticketBuilt({ ticket }).slice(0, 11)

package/engine/testing/wait-scenarios.ts

@@ -0,0 +1,186 @@
+import { ticketIssue } from './intake-fixtures'
+import {
+    journalRecords,
+    waitUntil,
+    type ManyTicketsScenario,
+} from './many-tickets'
+import {
+    APPROVE,
+    IMPLEMENTER_RESULT,
+    PRACTICE_SPEC_NUMBER,
+    practiceSpec,
+    SUM,
+    SUM_TEST,
+    TEST_WRITER_RESULT,
+} from './practice-repo'
+import { AVERAGE, AVERAGE_TEST } from './practice-run'
+
+import type { ScriptedTurn } from '../agents/scripted-launcher'
+import { createInMemoryTracker } from '../tracker/in-memory-tracker'
+
+/**
+ * A many-ticket run with all three waits between steps (#559): #21 "Add
+ * sum" and #22 "Add product" start from the same run-branch commit, so one
+ * waits for the other's baseline tests. #21's reviewer approves once #22's
+ * reviewer has started, and #22's reviewer approves once #21 has joined,
+ * so #22 is approved while #21's checks run on the run branch: it waits to
+ * join after #21. #23 "Add average" waits on #21; once #21 is pushed, #22
+ * is still in the join queue, so #23 starts once #22 has joined. No ticket
+ * touches another's files, so no join clashes.
+ */
+
+const PRODUCT_TEST = `import { describe, expect, test } from 'bun:test'
+
+import { product } from './product'
+
+describe('product', () => {
+    test('multiplies two numbers', () => {
+        expect(product({ numbers: [2, 3] })).toBe(6)
+    })
+
+    test('of no numbers is one', () => {
+        expect(product({ numbers: [] })).toBe(1)
+    })
+})
+`
+
+const PRODUCT = `export const product = ({ numbers }: { numbers: number[] }): number =>
+    numbers.reduce((total, each) => total * each, 1)
+`
+
+/** A test-writer's result mapping AC1 and AC2 to the tests of `file`. */
+const mapped = ({
+    file,
+    names,
+}: {
+    file: string
+    names: [string, string]
+}) => ({
+    outcome: 'tests_written',
+    criteria: names.map((name, index) => ({
+        criterion_id: `AC${index + 1}`,
+        tests: [{ file, name }],
+    })),
+    summary: 'One test per criterion.',
+    assumptions: [],
+    run_notes: [],
+})
+
+/** The baseline, join and start waits, in one run of #21, #22 and #23. */
+export const THREE_WAITS: ManyTicketsScenario = {
+    tracker: () =>
+        createInMemoryTracker({
+            issues: [
+                practiceSpec(),
+                ticketIssue({
+                    number: 21,
+                    title: 'Add sum',
+                    criteria: [
+                        'sum adds two numbers',
+                        'sum of no numbers is zero',
+                    ],
+                }),
+                ticketIssue({
+                    number: 22,
+                    title: 'Add product',
+                    criteria: [
+                        'product multiplies two numbers',
+                        'product of no numbers is one',
+                    ],
+                }),
+                ticketIssue({
+                    number: 23,
+                    title: 'Add average',
+                    criteria: [
+                        'average of two numbers is the one between them',
+                        'average of no numbers is zero',
+                    ],
+                    blocked_by_section: '- #21',
+                    blocked_by: [21],
+                }),
+            ],
+            sub_tickets: { [PRACTICE_SPEC_NUMBER]: [21, 22, 23] },
+        }),
+    turns: ({ journal_file }): ScriptedTurn[] => {
+        const reviewing = new Set<number>()
+        return [
+            {
+                role: 'test-writer',
+                ticket: 21,
+                files: { 'src/sum.test.ts': SUM_TEST },
+                result: TEST_WRITER_RESULT,
+            },
+            {
+                role: 'test-writer',
+                ticket: 22,
+                files: { 'src/product.test.ts': PRODUCT_TEST },
+                result: mapped({
+                    file: 'src/product.test.ts',
+                    names: [
+                        'product > multiplies two numbers',
+                        'product > of no numbers is one',
+                    ],
+                }),
+            },
+            {
+                role: 'implementer',
+                ticket: 21,
+                files: { 'src/sum.ts': SUM },
+                result: IMPLEMENTER_RESULT,
+            },
+            {
+                role: 'implementer',
+                ticket: 22,
+                files: { 'src/product.ts': PRODUCT },
+                result: { ...IMPLEMENTER_RESULT, summary: 'Added product.' },
+            },
+            {
+                role: 'ticket-reviewer',
+                ticket: 21,
+                act: async () =>
+                    waitUntil({
+                        check: () => reviewing.has(22),
+                        what: "#22's reviewer to start",
+                    }),
+                result: APPROVE,
+            },
+            {
+                role: 'ticket-reviewer',
+                ticket: 22,
+                act: async () => {
+                    reviewing.add(22)
+                    await waitUntil({
+                        check: () =>
+                            journalRecords(journal_file).some(
+                                (record) =>
+                                    record.kind === 'ticket_joined' &&
+                                    record.ticket === 21 &&
+                                    record.content.ok
+                            ),
+                        what: "#21's join",
+                    })
+                },
+                result: APPROVE,
+            },
+            {
+                role: 'test-writer',
+                ticket: 23,
+                files: { 'src/average.test.ts': AVERAGE_TEST },
+                result: mapped({
+                    file: 'src/average.test.ts',
+                    names: [
+                        'average > of two numbers is the one between them',
+                        'average > of no numbers is zero',
+                    ],
+                }),
+            },
+            {
+                role: 'implementer',
+                ticket: 23,
+                files: { 'src/average.ts': AVERAGE },
+                result: { ...IMPLEMENTER_RESULT, summary: 'Added average.' },
+            },
+            { role: 'ticket-reviewer', ticket: 23, result: APPROVE },
+        ]
+    },
+}