@alecsibilia/luca 14.0.0-alpha.2 → 14.0.0-alpha.21

package/engine/testing/run-budget-fixtures.ts

@@ -0,0 +1,165 @@
+/**
+ * Fixtures shared by the run budget decision tests (#435, #543): a run with
+ * a budget in its engine config, agent turns that count their tokens per
+ * model, and the run stuck on its budget with the spec issue told.
+ */
+import {
+    intakePassed,
+    practiceTicket,
+    runBranchCreated,
+    SESSIONS,
+    ticketBuilt,
+    withInstalls,
+} from './build-fixtures'
+import { recordsFrom } from './intake-fixtures'
+
+import type { AgentRole } from '../agents/role-results'
+import { decideSteps } from '../core/decide'
+import type { TicketSnapshot } from '../intake/intake-schemas'
+import type { JournalEntry } from '../journal/journal-record'
+
+export const SUM = practiceTicket({ number: 11, title: 'Add sum' })
+export const PRODUCT = practiceTicket({ number: 12, title: 'Add product' })
+
+/** The budget the tests give the run in its engine config. */
+export const BUDGET = 50_000
+
+type Tokens = {
+    input_tokens: number
+    output_tokens: number
+    cache_read_input_tokens: number
+    cache_creation_input_tokens: number
+}
+
+export const tokens = (
+    input: number,
+    output: number,
+    cache_read: number,
+    cache_creation: number
+): Tokens => ({
+    input_tokens: input,
+    output_tokens: output,
+    cache_read_input_tokens: cache_read,
+    cache_creation_input_tokens: cache_creation,
+})
+
+export const OPUS = 'claude-opus-5-5'
+export const HAIKU = 'claude-haiku-4-5-20251001'
+
+/** One agent turn's session, with its tokens per model. */
+export const turnSession = ({
+    ticket,
+    role,
+    model_usage,
+}: {
+    ticket: number | null
+    role: AgentRole
+    model_usage: Record<string, Tokens>
+}): JournalEntry => ({
+    kind: 'agent_session',
+    ticket,
+    role,
+    content: {
+        role,
+        session: {
+            session_id: SESSIONS[role],
+            usage: tokens(0, 0, 0, 0),
+            model_usage,
+            rate_limit_events: [],
+            billing_error: false,
+        },
+    },
+})
+
+/** A turn that counts exactly `counted` tokens, all output, on Opus. */
+export const turnOf = ({
+    ticket,
+    role,
+    counted,
+}: {
+    ticket: number
+    role: AgentRole
+    counted: number
+}): JournalEntry =>
+    turnSession({
+        ticket,
+        role,
+        model_usage: { [OPUS]: tokens(0, counted, 0, 0) },
+    })
+
+/**
+ * The decision step's actions for a run with these tickets and entries
+ * after intake. `budget` is the config's `run_budget_tokens`; leave it out
+ * for a config without one.
+ */
+export const stepsAfter = ({
+    tickets,
+    entries,
+    budget,
+}: {
+    tickets: TicketSnapshot[]
+    entries: JournalEntry[]
+    budget?: number
+}) =>
+    decideSteps({
+        records: recordsFrom({
+            entries: [
+                ...intakePassed({ tickets }).map(
+                    (entry): JournalEntry =>
+                        entry.kind === 'run_started' && budget !== undefined
+                            ? {
+                                  ...entry,
+                                  content: {
+                                      ...entry.content,
+                                      config: {
+                                          ...entry.content.config,
+                                          run_budget_tokens: budget,
+                                      },
+                                  },
+                              }
+                            : entry
+                ),
+                ...withInstalls({ entries }),
+            ],
+        }),
+    })
+
+export const oneSteps = (entries: JournalEntry[], budget: number | undefined) =>
+    stepsAfter({ tickets: [SUM], entries, budget })
+
+/** #11's worktree, baseline, and tests written: its red check is next. */
+export const sumWritten = (): JournalEntry[] =>
+    ticketBuilt({ ticket: 11 }).slice(0, 3)
+
+/** #11 with its tests written, and its test-writer's turn counting `counted`. */
+export const sumSpent = (counted: number): JournalEntry[] => [
+    runBranchCreated(),
+    ...sumWritten(),
+    turnOf({ ticket: 11, role: 'test-writer', counted }),
+]
+
+/** The run went stuck on its budget. */
+export const runStuck = (): JournalEntry => ({
+    kind: 'run_stuck',
+    ticket: null,
+    role: null,
+    content: {
+        reason: 'run_budget',
+        detail: `The run used ${BUDGET} tokens of its ${BUDGET} budget.`,
+    },
+})
+
+/** The engine told the spec issue the run is stuck, in comment 150. */
+export const runStuckReported = (): JournalEntry => ({
+    kind: 'stuck_reported',
+    ticket: null,
+    role: null,
+    content: { comment_id: 150, body: 'stuck' },
+})
+
+/** #11 used the whole budget; the run is stuck and the spec issue told. */
+export const told = (): JournalEntry[] => [
+    ...sumSpent(BUDGET),
+    runStuck(),
+    runStuckReported(),
+]

package/engine/testing/run-fixtures.ts

@@ -0,0 +1,219 @@
+/**
+ * Journal entry builders for a run's scaffolding: its config and tickets, its
+ * branch and worktrees, the installs and baseline tests, and the end of the
+ * run (worktrees removed, the PR opened).
+ */
+import { SPEC_OWNER } from './intake-fixtures'
+
+import type { EngineConfig } from '../config/engine-config'
+import type { TestRun } from '../gates/gate-schemas'
+import type { TicketSnapshot } from '../intake/intake-schemas'
+import type { JournalEntry } from '../journal/journal-record'
+
+export const BUILD_CONFIG: EngineConfig = {
+    checks: { test: 'bun test', types: 'bun run types', lint: 'bun run lint' },
+    test_file_patterns: ['**/*.test.ts'],
+    test_setup_files: [],
+    rule_files: [],
+}
+
+/** A ticket as intake snapshots it. */
+export const practiceTicket = ({
+    number,
+    title,
+    criteria,
+    labels,
+    blockers,
+}: {
+    number: number
+    title?: string
+    criteria?: string[]
+    /** Defaults to `ready-for-agent` only; add `refactor` for a refactor ticket. */
+    labels?: string[]
+    /** The tickets it waits on. Defaults to none. */
+    blockers?: number[]
+}): TicketSnapshot => ({
+    number,
+    title: title ?? 'Add sum',
+    body: '## What to build\n\nA sum function.',
+    labels: labels ?? ['ready-for-agent'],
+    url: `https://github.com/acme/app/issues/${number}`,
+    criteria: (criteria ?? ['sum adds two numbers']).map((text, index) => ({
+        id: `AC${index + 1}`,
+        text,
+    })),
+    blockers: blockers ?? [],
+})
+
+/** A run whose intake passed with these tickets, in this order. */
+export const intakePassed = ({
+    tickets,
+}: {
+    tickets: TicketSnapshot[]
+}): JournalEntry[] => [
+    {
+        kind: 'run_started',
+        ticket: null,
+        role: null,
+        content: { spec_number: 10, config: BUILD_CONFIG, base_branch: 'main' },
+    },
+    {
+        kind: 'spec_snapshot',
+        ticket: 10,
+        role: null,
+        content: {
+            spec: {
+                number: 10,
+                title: 'Practice spec',
+                body: '## Testing Decisions\n\n- Test the sum.',
+                labels: ['ready-for-agent'],
+                url: 'https://github.com/acme/app/issues/10',
+                author: SPEC_OWNER,
+            },
+            ticket_order: tickets.map((ticket) => ticket.number),
+            closed_tickets: [],
+        },
+    },
+    ...tickets.map(
+        (ticket): JournalEntry => ({
+            kind: 'ticket_snapshot',
+            ticket: ticket.number,
+            role: null,
+            content: ticket,
+        })
+    ),
+]
+
+export const RUN_BRANCH = 'luca/spec-10-run'
+
+/** Where fixtures put each ticket's worktree, and the run branch's. */
+export const ticketPath = (ticket: number): string =>
+    `/runs/run/tickets/${ticket}`
+
+export const RUN_BRANCH_PATH = '/runs/run/run-branch'
+
+export const runBranchCreated = (): JournalEntry => ({
+    kind: 'run_branch_created',
+    ticket: null,
+    role: null,
+    content: {
+        branch: RUN_BRANCH,
+        path: RUN_BRANCH_PATH,
+        base_sha: 'b0',
+    },
+})
+
+export const ticketWorktreeCreated = ({
+    ticket,
+}: {
+    ticket: number
+}): JournalEntry => ({
+    kind: 'ticket_worktree_created',
+    ticket,
+    role: null,
+    content: {
+        branch: `${RUN_BRANCH}--ticket-${ticket}`,
+        path: ticketPath(ticket),
+        base_sha: 'b0',
+    },
+})
+
+/**
+ * The engine's install in a new worktree: the run branch's checkout
+ * (`ticket` unset) or a ticket's worktree. Passing unless `ok` is false.
+ */
+export const dependenciesInstalled = ({
+    ticket,
+    ok,
+}: {
+    ticket?: number
+    ok?: boolean
+}): JournalEntry => ({
+    kind: 'dependencies_installed',
+    ticket: ticket ?? null,
+    role: null,
+    content: {
+        target: ticket === undefined ? 'run_branch' : 'ticket',
+        check: {
+            name: 'install',
+            command: 'bun install --frozen-lockfile',
+            ok: ok ?? true,
+            exit_code: ok === false ? 1 : 0,
+            output:
+                ok === false
+                    ? 'error: lockfile had changes, but lockfile is frozen'
+                    : '',
+        },
+    },
+})
+
+/**
+ * The entries with a passing install after each new worktree that has none,
+ * so tests about later steps need not list the installs.
+ */
+export const withInstalls = ({
+    entries,
+}: {
+    entries: JournalEntry[]
+}): JournalEntry[] =>
+    entries.flatMap((entry, index) => {
+        const made =
+            entry.kind === 'run_branch_created' ||
+            entry.kind === 'ticket_worktree_created'
+        if (!made || entries[index + 1]?.kind === 'dependencies_installed') {
+            return [entry]
+        }
+        return [
+            entry,
+            dependenciesInstalled({ ticket: entry.ticket ?? undefined }),
+        ]
+    })
+
+/** A test run with no test files, as a fresh repo has. */
+export const emptyTestRun = (): TestRun => ({
+    command: 'bun test',
+    ok: true,
+    exit_code: 1,
+    no_test_files: true,
+    cases: [],
+    files_without_results: [],
+    output: '0 test files matching',
+})
+
+export const baselineTests = ({
+    ticket,
+}: {
+    ticket: number
+}): JournalEntry => ({
+    kind: 'baseline_tests',
+    ticket,
+    role: null,
+    content: emptyTestRun(),
+})
+
+/** The engine removed these worktrees at the end of the run. */
+export const worktreesRemoved = ({
+    paths,
+}: {
+    paths: string[]
+}): JournalEntry => ({
+    kind: 'worktrees_removed',
+    ticket: null,
+    role: null,
+    content: { paths },
+})
+
+/** The run's one PR, #99, opened from the run branch. */
+export const pullRequestOpened = (): JournalEntry => ({
+    kind: 'pull_request_opened',
+    ticket: null,
+    role: null,
+    content: {
+        number: 99,
+        url: 'https://github.com/acme/app/pull/99',
+        head: RUN_BRANCH,
+        base: 'main',
+        title: 'Practice spec (#10)',
+        body: '',
+    },
+})

package/engine/testing/sdk-play.ts

@@ -0,0 +1,177 @@
+import { realpathSync } from 'node:fs'
+
+import type { HookJSONOutput, Options } from '@anthropic-ai/claude-agent-sdk'
+
+import type { AgentQuery } from '../agents/claude-launcher'
+import { fakeQuery, repo } from '../agents/claude-launcher-helpers'
+
+/**
+ * A fake SDK session for the launcher's tests: SDK-shaped messages played
+ * through `createClaudeLauncher`'s `query`, and tool calls played through
+ * the guard hooks the launch set up, as the SDK would. No model, no process.
+ */
+
+/** A step: an SDK message, or work to do once the launcher has read the message before it. */
+export type Step = unknown | ((options: Options) => Promise<void>)
+
+/**
+ * A fake `query` that plays `steps`: each message is yielded in turn, and
+ * each function runs after the launcher has read the message before it,
+ * the way the agent's tool calls happen mid-turn.
+ */
+export const playing = ({ steps }: { steps: Step[] }): AgentQuery => {
+    const messages: unknown[] = []
+    const after = new Map<unknown, ((options: Options) => Promise<void>)[]>()
+    for (const step of steps) {
+        if (typeof step === 'function') {
+            const last = messages.at(-1)
+            after.set(last, [
+                ...(after.get(last) ?? []),
+                step as (options: Options) => Promise<void>,
+            ])
+        } else messages.push(step)
+    }
+    const fake = fakeQuery({ messages })
+    return (params) => {
+        const inner = fake.query(params)
+        return {
+            ...inner,
+            [Symbol.asyncIterator]: async function* () {
+                for await (const message of inner) {
+                    yield message
+                    for (const work of after.get(message) ?? []) {
+                        await work(params.options)
+                    }
+                }
+            },
+        }
+    }
+}
+
+/** One message's token counts, as the SDK reports them. */
+export type Usage = {
+    input_tokens: number
+    output_tokens: number
+    cache_read_input_tokens: number
+    cache_creation_input_tokens: number
+}
+
+/**
+ * One API response, as the SDK streams it: the agent's (`parent` null), or
+ * a helper's (`parent` the `Agent` call that started it).
+ */
+export const assistant = ({
+    id,
+    content,
+    parent = null,
+    model = 'claude-opus-5-5',
+    usage = {
+        input_tokens: 10,
+        output_tokens: 10,
+        cache_read_input_tokens: 0,
+        cache_creation_input_tokens: 0,
+    },
+}: {
+    id: string
+    content: unknown[]
+    parent?: string | null
+    model?: string
+    usage?: Usage
+}) => ({
+    type: 'assistant',
+    message: {
+        id,
+        type: 'message',
+        role: 'assistant',
+        model,
+        content,
+        stop_reason: 'tool_use',
+        usage,
+    },
+    parent_tool_use_id: parent,
+    session_id: 'session-1',
+    uuid: `uuid-${id}`,
+})
+
+/** A tool call inside an `assistant` message's content. */
+export const toolUse = (
+    id: string,
+    name: string,
+    input: Record<string, unknown>
+) => ({ type: 'tool_use', id, name, input })
+
+/** A tool's result, handed back to the agent (`parent` null) or a helper. */
+export const toolResult = ({
+    tool_use_id,
+    content,
+    parent = null,
+}: {
+    tool_use_id: string
+    content: string
+    parent?: string | null
+}) => ({
+    type: 'user',
+    message: {
+        role: 'user',
+        content: [{ type: 'tool_result', tool_use_id, content }],
+    },
+    parent_tool_use_id: parent,
+    session_id: 'session-1',
+})
+
+/**
+ * Plays one tool call through every `PreToolUse` hook the launch set up
+ * whose matcher fits the tool, as the SDK would, and gives the first
+ * denial, or `{}` when every hook let it through. With `helper`, the call
+ * comes from inside a helper (the SDK's `agent_id` and `agent_type`). `cwd`
+ * is the test's repo, its real path, unless told.
+ */
+export const preToolUse = async ({
+    options,
+    tool_name,
+    tool_input,
+    tool_use_id,
+    helper = false,
+    cwd = realpathSync(repo),
+}: {
+    options: Options
+    tool_name: string
+    tool_input: Record<string, unknown>
+    tool_use_id: string
+    helper?: boolean
+    cwd?: string
+}): Promise<HookJSONOutput> => {
+    const signal = new AbortController().signal
+    for (const matcher of options.hooks?.PreToolUse ?? []) {
+        if (
+            matcher.matcher !== undefined &&
+            !new RegExp(`^(?:${matcher.matcher})$`).test(tool_name)
+        )
+            continue
+        for (const hook of matcher.hooks) {
+            const answer = await hook(
+                {
+                    hook_event_name: 'PreToolUse',
+                    tool_name,
+                    tool_input,
+                    tool_use_id,
+                    session_id: 'session-1',
+                    transcript_path: '/dev/null',
+                    cwd,
+                    ...(helper
+                        ? { agent_id: 'helper-1', agent_type: 'searcher' }
+                        : {}),
+                },
+                tool_use_id,
+                { signal }
+            )
+            if (
+                'hookSpecificOutput' in answer &&
+                answer.hookSpecificOutput?.hookEventName === 'PreToolUse' &&
+                answer.hookSpecificOutput.permissionDecision === 'deny'
+            )
+                return answer
+        }
+    }
+    return {}
+}