@alecsibilia/luca 14.0.0-alpha.2 → 14.0.0-alpha.21

package/engine/shell/run-command.ts

@@ -1,3 +1,10 @@
+/** A process the command started that still ran after a timeout's kill. */
+export type Survivor = {
+    pid: number
+    /** Its command line, from `ps`. */
+    command: string
+}
+
 /** What a finished command left behind. */
 export type CommandResult = {
     /** `null` when the command was killed. */
@@ -5,6 +12,8 @@ export type CommandResult = {
     stdout: string
     stderr: string
     timed_out: boolean
+    /** Set on a timeout only: what still ran after the kill (#571). */
+    survivors?: Survivor[]
 }
 
 /** Commands the engine runs give up after this long unless told otherwise. */
@@ -12,8 +21,9 @@ export const DEFAULT_COMMAND_TIMEOUT_MS = 300_000
 
 /**
  * After a timed-out command is killed, how long the engine still waits for
- * its output. A child that left the command's process group can hold the
- * output pipe open; past this, the engine stops reading and moves on.
+ * its output when the caller doesn't say (`pipe_grace_ms`). A child that left
+ * the command's process group can hold the output pipe open; past this, the
+ * engine stops reading and moves on.
  */
 const PIPE_GRACE_MS = 2_000
 
@@ -39,6 +49,110 @@ const signalGroup = ({
     }
 }
 
+/** One line of `ps`: a process, its parent, and its process group. */
+type ProcessEntry = Survivor & { ppid: number; pgid: number }
+
+/**
+ * Every process now running, from `ps`. Empty when `ps` fails: then the
+ * engine can only kill the command's own group.
+ */
+const listProcessTable = async (): Promise<ProcessEntry[]> => {
+    try {
+        const ps = Bun.spawn(
+            ['ps', '-axww', '-o', 'pid=,ppid=,pgid=,command='],
+            {
+                stdin: 'ignore',
+                stdout: 'pipe',
+                stderr: 'ignore',
+            }
+        )
+        const [text, code] = await Promise.all([
+            new Response(ps.stdout).text(),
+            ps.exited,
+        ])
+        if (code !== 0) return []
+        return text.split('\n').flatMap((line) => {
+            const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s(.*)$/.exec(line)
+            if (match === null) return []
+            const [, pid, ppid, pgid, command] = match
+            return [
+                {
+                    pid: Number(pid),
+                    ppid: Number(ppid),
+                    pgid: Number(pgid),
+                    command: (command ?? '').trim(),
+                },
+            ]
+        })
+    } catch {
+        return []
+    }
+}
+
+/** Every process below `pid`, found by parent process id. */
+const descendantsOf = async (pid: number): Promise<ProcessEntry[]> => {
+    const table = await listProcessTable()
+    const found: ProcessEntry[] = []
+    const parents = new Set([pid])
+    for (let grew = true; grew; ) {
+        grew = false
+        for (const entry of table) {
+            if (!parents.has(entry.ppid) || parents.has(entry.pid)) continue
+            parents.add(entry.pid)
+            found.push(entry)
+            grew = true
+        }
+    }
+    return found
+}
+
+/** Whether a process with this id still runs. */
+const isRunning = (pid: number): boolean => {
+    try {
+        process.kill(pid, 0)
+        return true
+    } catch {
+        return false
+    }
+}
+
+/** How long the engine waits for killed processes to go before naming them. */
+const SURVIVOR_WAIT_MS = 500
+
+/** Of `processes`, those still running after up to `SURVIVOR_WAIT_MS`. */
+const survivorsOf = async (processes: ProcessEntry[]): Promise<Survivor[]> => {
+    const end = Date.now() + SURVIVOR_WAIT_MS
+    let alive = processes.filter(({ pid }) => isRunning(pid))
+    while (alive.length > 0 && Date.now() < end) {
+        await Bun.sleep(25)
+        alive = alive.filter(({ pid }) => isRunning(pid))
+    }
+    return alive.map(({ pid, command }) => ({ pid, command }))
+}
+
+/**
+ * Kills a timed-out command and everything it started (#571). It first walks
+ * the process tree from the command, while the tree is still whole, then
+ * kills the command's group and the group of every process below it, so a
+ * child that started a group of its own goes too. Returns what it found.
+ */
+const killTree = async (pid: number): Promise<ProcessEntry[]> => {
+    const descendants = await descendantsOf(pid)
+    signalGroup({ pid, signal: 'SIGKILL' })
+    for (const entry of descendants) {
+        // Never the engine's own group.
+        if (entry.pgid > 1 && entry.pgid !== process.pid) {
+            signalGroup({ pid: entry.pgid, signal: 'SIGKILL' })
+        }
+        try {
+            process.kill(entry.pid, 'SIGKILL')
+        } catch {
+            // Gone already.
+        }
+    }
+    return descendants
+}
+
 /** Sends `signal` to every command still running. */
 const signalLiveGroups = (signal: NodeJS.Signals): void => {
     for (const pid of liveGroups) signalGroup({ pid, signal })
@@ -97,10 +211,11 @@ const collectText = (
  * Runs a command and collects its output. Never throws on a non-zero exit.
  *
  * The command runs in its own process group. One that runs past
- * `timeout_ms` is killed with everything it started (the whole group) and
- * marked `timed_out`, with `exit_code: null`. The engine then waits at most
- * a moment more for its output, so a child that left the group can't hold
- * the engine.
+ * `timeout_ms` is killed with everything it started (its group, and the
+ * group of every process below it) and marked `timed_out`, with
+ * `exit_code: null` and the `survivors` the kill missed. The engine then
+ * waits at most a moment more for its output (`pipe_grace_ms`, default
+ * `PIPE_GRACE_MS`), so a child that escaped the kill can't hold the engine.
  *
  * @example
  * const result = await runCommand({ cmd: ['git', 'status'], cwd })
@@ -111,6 +226,7 @@ export const runCommand = async ({
     cwd,
     timeout_ms,
     env,
+    pipe_grace_ms,
 }: {
     cmd: string[]
     cwd: string
@@ -118,6 +234,11 @@ export const runCommand = async ({
     timeout_ms?: number
     /** Variables set on top of the engine's own environment. */
     env?: Record<string, string>
+    /**
+     * How long to keep reading output after a timeout kill; defaults to
+     * `PIPE_GRACE_MS`. Tests of that wait pass a short one (#580).
+     */
+    pipe_grace_ms?: number
 }): Promise<CommandResult> => {
     stopCommandsOnExit()
     const proc = Bun.spawn({
@@ -133,16 +254,17 @@ export const runCommand = async ({
     liveGroups.add(proc.pid)
     const stdout = collectText(proc.stdout)
     const stderr = collectText(proc.stderr)
-    let timedOut = false
+    let killed: Promise<ProcessEntry[]> | undefined
     let grace: ReturnType<typeof setTimeout> | undefined
     const timer = setTimeout(() => {
-        timedOut = true
-        signalGroup({ pid: proc.pid, signal: 'SIGKILL' })
-        proc.kill('SIGKILL')
-        grace = setTimeout(() => {
-            stdout.stop()
-            stderr.stop()
-        }, PIPE_GRACE_MS)
+        killed = killTree(proc.pid).then((descendants) => {
+            proc.kill('SIGKILL')
+            grace = setTimeout(() => {
+                stdout.stop()
+                stderr.stop()
+            }, pipe_grace_ms ?? PIPE_GRACE_MS)
+            return descendants
+        })
     }, timeout_ms ?? DEFAULT_COMMAND_TIMEOUT_MS)
     const [out, err, exitCode] = await Promise.all([
         stdout.text,
@@ -150,14 +272,18 @@ export const runCommand = async ({
         proc.exited,
     ])
     clearTimeout(timer)
+    const descendants = await killed
     clearTimeout(grace)
     liveGroups.delete(proc.pid)
-    return {
-        exit_code: timedOut ? null : exitCode,
+    const result = {
+        exit_code: killed === undefined ? exitCode : null,
         stdout: out,
         stderr: err,
-        timed_out: timedOut,
+        timed_out: killed !== undefined,
     }
+    return descendants === undefined
+        ? result
+        : { ...result, survivors: await survivorsOf(descendants) }
 }
 
 /** Runs a command line from the engine config through `sh -c`. */
@@ -165,12 +291,55 @@ export const runShell = ({
     command,
     cwd,
     timeout_ms,
+    pipe_grace_ms,
 }: {
     command: string
     cwd: string
     timeout_ms?: number
+    /** See `runCommand`. */
+    pipe_grace_ms?: number
 }): Promise<CommandResult> =>
-    runCommand({ cmd: ['sh', '-c', command], cwd, timeout_ms })
+    runCommand({
+        cmd: ['sh', '-c', command],
+        cwd,
+        timeout_ms,
+        pipe_grace_ms,
+    })
+
+/** A time limit in words: `30 minutes`, `1 minute`, `0.3 seconds`. */
+export const durationText = (ms: number): string => {
+    const [amount, unit] =
+        ms >= 60_000 ? [ms / 60_000, 'minute'] : [ms / 1000, 'second']
+    const rounded = Math.round(amount * 10) / 10
+    return `${rounded} ${unit}${rounded === 1 ? '' : 's'}`
+}
+
+/**
+ * The line a timed-out command's output starts with: how long it had, that
+ * the engine stopped it, then `hint`, if any. With `survivors`, it names
+ * each one instead of claiming everything was stopped (#571).
+ *
+ * @example
+ * timedOutLine({ timeout_ms: 300_000, hint: 'To give it longer, raise `test_timeout_ms`.' })
+ * // 'Timed out after 5 minutes, so the engine stopped it and everything it started. To give it longer, raise `test_timeout_ms`.\n'
+ */
+export const timedOutLine = ({
+    timeout_ms,
+    hint,
+    survivors,
+}: {
+    timeout_ms: number
+    /** How to give the command longer. */
+    hint?: string
+    /** What still ran after the kill. */
+    survivors?: Survivor[]
+}): string => {
+    const stopped =
+        survivors === undefined || survivors.length === 0
+            ? 'so the engine stopped it and everything it started.'
+            : `so the engine stopped it. These were still running after the kill: ${survivors.map(({ pid, command }) => `pid ${pid} (${command})`).join(', ')}.`
+    return `Timed out after ${durationText(timeout_ms)}, ${stopped}${hint === undefined ? '' : ` ${hint}`}\n`
+}
 
 /** Keeps the start and end of long output, for the journal and prompts. */
 export const clipOutput = ({

package/engine/testing/agent-turn-fixtures.ts

@@ -0,0 +1,287 @@
+/**
+ * Journal entry builders for agent turns: a turn started, its result for
+ * each role (tests written, implemented, reviewed), a failed turn, and the
+ * launcher's summary of a turn.
+ */
+import type {
+    AgentRole,
+    CriterionTests,
+    Finding,
+    FindingResponse,
+    FindingRuling,
+} from '../agents/role-results'
+import type { JournalEntry } from '../journal/journal-record'
+
+/** The session id fixtures give each role's turns unless told otherwise. */
+export const SESSIONS = {
+    'test-writer': 'tw-1',
+    implementer: 'impl-1',
+    'ticket-reviewer': 'rev-1',
+    'architecture-lens': 'architecture-1',
+    'simplification-lens': 'simplification-1',
+    'security-lens': 'security-1',
+    'integration-lens': 'integration-1',
+    'rules-lens': 'rules-1',
+    learner: 'learner-1',
+} as const satisfies Record<AgentRole, string>
+
+/** An agent turn started: a fresh launch, or a follow-up to `follow_up_of`. */
+export const agentStarted = ({
+    ticket,
+    role,
+    prompt,
+    follow_up_of,
+}: {
+    /** `null` for the final review. */
+    ticket: number | null
+    role: AgentRole
+    prompt?: string
+    follow_up_of?: string
+}): JournalEntry => ({
+    kind: 'agent_started',
+    ticket,
+    role,
+    content: {
+        role,
+        prompt: prompt ?? 'p',
+        follow_up_of: follow_up_of ?? null,
+    },
+})
+
+export const testsWritten = ({
+    ticket,
+    session_id,
+    assumptions,
+    finding_responses,
+    run_notes,
+    criteria,
+}: {
+    /** `null` for the final review. */
+    ticket: number | null
+    /** Defaults to `SESSIONS['test-writer']`. */
+    session_id?: string
+    /** Defaults to one test, `sum adds two numbers`, for AC1. */
+    criteria?: CriterionTests[]
+    /** Defaults to one assumption. */
+    assumptions?: string[]
+    /** A review fixer's answer to each finding it got. */
+    finding_responses?: FindingResponse[]
+    /** Defaults to none. */
+    run_notes?: string[]
+}): JournalEntry => ({
+    kind: 'agent_finished',
+    ticket,
+    role: 'test-writer',
+    content: {
+        role: 'test-writer',
+        session_id: session_id ?? SESSIONS['test-writer'],
+        result: {
+            outcome: 'tests_written',
+            criteria: criteria ?? [
+                {
+                    criterion_id: 'AC1',
+                    tests: [
+                        {
+                            file: 'src/sum.test.ts',
+                            name: 'sum adds two numbers',
+                        },
+                    ],
+                },
+            ],
+            summary: 'One test for AC1.',
+            assumptions: assumptions ?? ['Numbers are integers.'],
+            run_notes: run_notes ?? [],
+            finding_responses: finding_responses ?? [],
+        },
+    },
+})
+
+/** A test-writer that found nothing new to test. */
+export const nothingNewToTest = ({
+    ticket,
+}: {
+    ticket: number
+}): JournalEntry => ({
+    kind: 'agent_finished',
+    ticket,
+    role: 'test-writer',
+    content: {
+        role: 'test-writer',
+        session_id: SESSIONS['test-writer'],
+        result: { outcome: 'nothing_new_to_test' },
+    },
+})
+
+export const implemented = ({
+    ticket,
+    outcome,
+    session_id,
+    reason,
+    assumptions,
+    finding_responses,
+    run_notes,
+}: {
+    /** `null` for the final review. */
+    ticket: number | null
+    outcome?: 'done' | 'bad_test'
+    /** Defaults to `SESSIONS.implementer`. */
+    session_id?: string
+    /** The bad test's reason. Defaults to "Wrong sum." */
+    reason?: string
+    assumptions?: string[]
+    finding_responses?: FindingResponse[]
+    /** Defaults to none. */
+    run_notes?: string[]
+}): JournalEntry => ({
+    kind: 'agent_finished',
+    ticket,
+    role: 'implementer',
+    content: {
+        role: 'implementer',
+        session_id: session_id ?? SESSIONS.implementer,
+        result: {
+            outcome: outcome ?? 'done',
+            bad_test:
+                outcome === 'bad_test'
+                    ? {
+                          file: 'src/sum.test.ts',
+                          name: 'sum adds two numbers',
+                          reason: reason ?? 'Wrong sum.',
+                      }
+                    : null,
+            summary: 'Added sum.',
+            assumptions: assumptions ?? [],
+            run_notes: run_notes ?? [],
+            finding_responses: finding_responses ?? [],
+        },
+    },
+})
+
+/** One reviewer finding; a code should-fix in `src/sum.ts` unless told otherwise. */
+export const finding = ({
+    id,
+    severity,
+    kind,
+    title,
+    file,
+}: {
+    id: string
+    severity?: 'blocker' | 'should_fix' | 'nit'
+    kind?: 'code' | 'test'
+    title?: string
+    file?: string | null
+}): Finding => ({
+    id,
+    severity: severity ?? 'should_fix',
+    kind: kind ?? 'code',
+    file: file === undefined ? 'src/sum.ts' : file,
+    title: title ?? `Finding ${id}`,
+    detail: `Detail of ${id}.`,
+})
+
+/**
+ * A ticket reviewer's result. The verdict follows the findings: changes are
+ * requested when any is a blocker or should-fix.
+ */
+export const reviewed = ({
+    ticket,
+    findings,
+    rulings,
+    session_id,
+}: {
+    ticket: number
+    findings?: Finding[]
+    rulings?: FindingRuling[]
+    /** Defaults to `SESSIONS['ticket-reviewer']`. */
+    session_id?: string
+}): JournalEntry => {
+    const list = findings ?? []
+    const blocking = list.some(({ severity }) => severity !== 'nit')
+    return {
+        kind: 'agent_finished',
+        ticket,
+        role: 'ticket-reviewer',
+        content: {
+            role: 'ticket-reviewer',
+            session_id: session_id ?? SESSIONS['ticket-reviewer'],
+            result: {
+                verdict: blocking ? 'changes_requested' : 'approve',
+                findings: list,
+                rulings: rulings ?? [],
+                summary: 'Looks right.',
+                assumptions: [],
+            },
+        },
+    }
+}
+
+/** An agent turn failed, in the session its role's fixtures use unless told otherwise. */
+export const agentFailed = ({
+    ticket,
+    role,
+    failure,
+    error,
+    session_id,
+    session_gone,
+    out_of_turns,
+}: {
+    /** `null` for the final review. */
+    ticket: number | null
+    role: AgentRole
+    failure: 'agent' | 'result' | 'guard' | 'engine'
+    /** Defaults to "It broke." */
+    error?: string
+    /** Defaults to `SESSIONS[role]`; `null` for a turn with no session. */
+    session_id?: string | null
+    /** A follow-up found its session closed or gone (#521). Left out by default. */
+    session_gone?: boolean
+    /** The turn reached its role's turn limit (#543). Left out by default. */
+    out_of_turns?: boolean
+}): JournalEntry => ({
+    kind: 'agent_failed',
+    ticket,
+    role,
+    content: {
+        role,
+        failure,
+        error: error ?? 'It broke.',
+        session_id: session_id === undefined ? SESSIONS[role] : session_id,
+        ...(session_gone === undefined ? {} : { session_gone }),
+        ...(out_of_turns === undefined ? {} : { out_of_turns }),
+    },
+})
+
+/** The launcher's summary of one agent turn, with these readings and tokens. */
+export const agentSession = ({
+    ticket,
+    role,
+    rate_limit_events,
+    billing_error,
+    output_tokens,
+}: {
+    /** `null` for the final review. */
+    ticket: number | null
+    role: AgentRole
+    rate_limit_events?: Record<string, unknown>[]
+    billing_error?: boolean
+    /** Defaults to 100; input tokens are always 10. */
+    output_tokens?: number
+}): JournalEntry => ({
+    kind: 'agent_session',
+    ticket,
+    role,
+    content: {
+        role,
+        session: {
+            session_id: SESSIONS[role],
+            usage: {
+                input_tokens: 10,
+                output_tokens: output_tokens ?? 100,
+                cache_read_input_tokens: 0,
+                cache_creation_input_tokens: 0,
+            },
+            rate_limit_events: rate_limit_events ?? [],
+            billing_error: billing_error ?? false,
+        },
+    },
+})

package/engine/testing/already-done-fixtures.ts

@@ -0,0 +1,97 @@
+/**
+ * Journal entry builders for a ticket whose work is already on the base
+ * branch: the test-writer's evidence, the engine's check of it, and the
+ * ticket noted and closed.
+ */
+import { SESSIONS } from './agent-turn-fixtures'
+
+import type { JournalEntry } from '../journal/journal-record'
+
+/** The commit fixtures say did an already-done ticket's work. */
+export const DONE_SHA = '3559c25f5a1b2c3d4e5f60718293a4b5c6d7e8f9'
+
+/**
+ * A test-writer that found the ticket's work already on the base branch:
+ * done by `DONE_SHA` unless told otherwise, with one existing test for AC1.
+ */
+export const alreadyDone = ({
+    ticket,
+    shas,
+}: {
+    ticket: number
+    /** Defaults to `[DONE_SHA]`. */
+    shas?: string[]
+}): JournalEntry => ({
+    kind: 'agent_finished',
+    ticket,
+    role: 'test-writer',
+    content: {
+        role: 'test-writer',
+        session_id: SESSIONS['test-writer'],
+        result: {
+            outcome: 'already_done',
+            done_by: (shas ?? [DONE_SHA]).map((sha) => ({
+                sha,
+                title: 'feat: add sum',
+            })),
+            criteria: [
+                {
+                    criterion_id: 'AC1',
+                    tests: [
+                        {
+                            file: 'src/sum.test.ts',
+                            name: 'sum adds two numbers',
+                        },
+                    ],
+                },
+            ],
+            summary: 'The sum is already on main, with its test.',
+        },
+    },
+})
+
+/**
+ * The engine checked a test-writer's "already done" evidence (#495): it
+ * passed, or, with `problems`, it did not.
+ */
+export const alreadyDoneChecked = ({
+    ticket,
+    problems,
+}: {
+    ticket: number
+    /** What did not check out. Left out, none: the check passed. */
+    problems?: string[]
+}): JournalEntry => ({
+    kind: 'already_done_checked',
+    ticket,
+    role: null,
+    content: {
+        ok: (problems ?? []).length === 0,
+        problems: problems ?? [],
+        base_sha: 'b0',
+        tests: null,
+    },
+})
+
+/** The engine noted that a ticket's work is already done, on the spec issue. */
+export const ticketAlreadyDone = ({
+    ticket,
+    shas,
+}: {
+    ticket: number
+    /** Defaults to `[DONE_SHA]`. */
+    shas?: string[]
+}): JournalEntry => ({
+    kind: 'ticket_already_done',
+    ticket,
+    role: null,
+    content: { shas: shas ?? [DONE_SHA], comment_id: 300 + ticket },
+})
+
+/** The engine closed an already-done ticket, with a comment on it. */
+export const ticketClosed = ({ ticket }: { ticket: number }): JournalEntry => ({
+    kind: 'ticket_closed',
+    ticket,
+    role: null,
+    content: { comment_id: 400 + ticket },
+})