@alecsibilia/luca 14.0.0-alpha.2 → 14.0.0-alpha.21

package/engine/journal/agent-turn-replay.ts

@@ -0,0 +1,203 @@
+/**
+ * Small pure reads of an agent turn, shared by ticket and final review
+ * replay: a finished agent's assumptions and bad test, the sessions after a
+ * turn, the out-of-turns counts, the fixer roles, a fix round's answers and `other_role` hand-offs,
+ * and the "won't fix" answers a review let go.
+ */
+import omit from 'lodash/omit'
+import uniq from 'lodash/uniq'
+
+import type { JournalRecord } from './journal-record'
+import type { DeclinedFinding, ReviewFix, WriterRole } from './ticket-progress'
+
+import type {
+    AgentRole,
+    BadTest,
+    FindingResponse,
+    ImplementerResult,
+    TicketReviewResult,
+} from '../agents/role-results'
+import { DEFAULT_MODEL } from '../config/engine-config'
+
+export type FinishedContent = Extract<
+    JournalRecord,
+    { kind: 'agent_finished' }
+>['content']
+
+/** A finished agent's assumptions; the learner makes none. */
+export const assumptionsOf = (finished: FinishedContent): string[] =>
+    'assumptions' in finished.result ? finished.result.assumptions : []
+
+/** A fixer's bad test, or the implementer's summary as its reason. */
+export const badTestOf = (result: ImplementerResult): BadTest | null =>
+    result.outcome === 'bad_test'
+        ? (result.bad_test ?? { file: '', name: '', reason: result.summary })
+        : null
+
+/** The sessions after a turn: the role's latest, or none if unknown. */
+export const sessionsAfter = ({
+    sessions,
+    role,
+    session_id,
+}: {
+    sessions: Partial<Record<AgentRole, string>>
+    role: AgentRole
+    session_id: string | null
+}): Partial<Record<AgentRole, string>> => {
+    const rest = omit(sessions, role)
+    return session_id === null ? rest : { ...rest, [role]: session_id }
+}
+
+/** The out-of-turns counts after a failed turn (#543): one more for its role if it reached its limit. */
+export const outOfTurnsAfter = ({
+    counts,
+    failed,
+}: {
+    counts: Partial<Record<AgentRole, number>>
+    failed: { role: AgentRole; out_of_turns: boolean }
+}): Partial<Record<AgentRole, number>> =>
+    failed.out_of_turns
+        ? { ...counts, [failed.role]: (counts[failed.role] ?? 0) + 1 }
+        : counts
+
+export const isFixerRole = (role: AgentRole): boolean =>
+    role === 'test-writer' || role === 'implementer'
+
+const OTHER_ROLE: Record<WriterRole, WriterRole> = {
+    'test-writer': 'implementer',
+    implementer: 'test-writer',
+}
+
+type FixAnswers = Pick<ReviewFix, 'findings' | 'responses' | 'handed_off'>
+
+/**
+ * A fixer's answers in its fix round, kept, with the round's findings it
+ * answered `other_role` handed to the other writer role (#616).
+ */
+export const fixerAnswered = ({
+    fix,
+    role,
+    responses,
+}: {
+    fix: FixAnswers
+    role: WriterRole
+    responses: FindingResponse[]
+}): Pick<ReviewFix, 'responses' | 'handed_off'> => {
+    const handed = responses.flatMap(({ finding_id, response }) =>
+        response === 'other_role' &&
+        fix.findings.some(({ id }) => id === finding_id)
+            ? [finding_id]
+            : []
+    )
+    const other = OTHER_ROLE[role]
+    return {
+        responses: [...fix.responses, ...responses],
+        handed_off:
+            handed.length === 0
+                ? fix.handed_off
+                : {
+                      ...fix.handed_off,
+                      [other]: uniq([
+                          ...(fix.handed_off[other] ?? []),
+                          ...handed,
+                      ]),
+                  },
+    }
+}
+
+/**
+ * The other role's answers to the findings handed to it (#616). Each one
+ * replaces the first fixer's `other_role`, and is final: a second
+ * `other_role` counts as `wont_fix`. Answers to findings not handed to it
+ * are dropped.
+ */
+export const handoffAnswered = ({
+    fix,
+    role,
+    responses,
+}: {
+    fix: FixAnswers
+    role: WriterRole
+    responses: FindingResponse[]
+}): Pick<ReviewFix, 'responses' | 'handed_off'> => {
+    const ids = fix.handed_off[role] ?? []
+    const answers = responses.flatMap((answer) => {
+        if (!ids.includes(answer.finding_id)) return []
+        return answer.response === 'other_role'
+            ? [{ ...answer, response: 'wont_fix' as const }]
+            : [answer]
+    })
+    const answered = new Set(answers.map(({ finding_id }) => finding_id))
+    return {
+        responses: [
+            ...fix.responses.filter(
+                ({ finding_id }) => !answered.has(finding_id)
+            ),
+            ...answers,
+        ],
+        handed_off: omit(fix.handed_off, role),
+    }
+}
+
+/**
+ * The writer role a fix round hands findings to next, once its fixers
+ * answered: the test-writer first, then the implementer. `null` with none,
+ * or while a fixer has yet to answer.
+ */
+export const handoffRole = ({
+    fix,
+}: {
+    fix: Pick<ReviewFix, 'handed_off' | 'tests_answered' | 'code_answered'>
+}): WriterRole | null => {
+    if (!fix.tests_answered || !fix.code_answered) return null
+    return (
+        (['test-writer', 'implementer'] as const).find(
+            (role) => (fix.handed_off[role] ?? []).length > 0
+        ) ?? null
+    )
+}
+
+/**
+ * Whether a fixer's answer declines its finding: `wont_fix`, or an
+ * `other_role` the other role never answered (#616).
+ */
+export const isDeclined = ({ response }: FindingResponse): boolean =>
+    response === 'wont_fix' || response === 'other_role'
+
+/**
+ * Whether an agent launched on a model cheaper than Opus (#604). Older
+ * journals name no model, and ran Opus.
+ */
+export const ranCheaper = (model: string | null | undefined): boolean =>
+    model !== null && model !== undefined && model !== DEFAULT_MODEL
+
+/**
+ * The fixers' "won't fix" answers the new review let go: its ruling was
+ * `accepted`, or it gave none and did not list the finding again.
+ */
+export const declinedBy = ({
+    fix,
+    review,
+}: {
+    fix: Pick<ReviewFix, 'findings' | 'responses'> | null
+    review: TicketReviewResult
+}): DeclinedFinding[] => {
+    if (fix === null) return []
+    const listed = new Set(review.findings.map(({ id }) => id))
+    return fix.responses.flatMap((answer) => {
+        const { finding_id, reason } = answer
+        if (!isDeclined(answer)) return []
+        const finding = fix.findings.find(({ id }) => id === finding_id)
+        if (finding === undefined) return []
+        const ruling = review.rulings.find(
+            (entry) => entry.finding_id === finding_id
+        )
+        const accepted =
+            ruling === undefined
+                ? !listed.has(finding_id)
+                : ruling.ruling === 'accepted'
+        return accepted
+            ? [{ finding, reason, ruling: ruling?.reason ?? '' }]
+            : []
+    })
+}

package/engine/journal/already-done-entries.ts

@@ -0,0 +1,55 @@
+import { z } from 'zod'
+
+import { ENTRY_FIELDS } from './entry-fields'
+
+import { TestRunSchema } from '../gates/gate-schemas'
+
+/**
+ * The journal entries of a ticket whose work is already on the base branch
+ * (#484): the evidence check, the ticket counted done, and its closing.
+ */
+
+/**
+ * The engine checked a test-writer's `already_done` evidence (#495) in the
+ * ticket's worktree, at `base_sha`, the commit it started from: each named
+ * commit is on the base, and each named test exists and passes there.
+ * `problems` says what did not check out; `tests` is the run of the named
+ * test files (`null` if none ran). A failed check is a failed try for the
+ * test-writer.
+ */
+export const AlreadyDoneCheckedEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('already_done_checked'),
+    content: z.object({
+        ok: z.boolean(),
+        problems: z.array(z.string()),
+        base_sha: z.string(),
+        tests: TestRunSchema.nullable(),
+    }),
+})
+
+/**
+ * A ticket whose work is already on the base branch (#484): its
+ * test-writer answered `already_done`, and `shas` are the commits that did
+ * the work. The ticket counts as done, not stuck, and tickets that wait on
+ * it go on. The engine said so on the spec issue, in its comment
+ * `comment_id`. The run's PR closes it; with no PR, `ticket_closed` follows.
+ */
+export const TicketAlreadyDoneEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('ticket_already_done'),
+    content: z.object({
+        shas: z.array(z.string().min(1)),
+        comment_id: z.number().int(),
+    }),
+})
+
+/**
+ * The engine closed an already-done ticket itself, as the run opens no PR
+ * to close it, after its comment `comment_id` on the ticket said why.
+ */
+export const TicketClosedEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('ticket_closed'),
+    content: z.object({ comment_id: z.number().int() }),
+})

package/engine/journal/commit-entries.ts

@@ -0,0 +1,56 @@
+import { z } from 'zod'
+
+import { ENTRY_FIELDS } from './entry-fields'
+
+import { LeftoverHitSchema, LeftoverPassSchema } from '../gates/gate-schemas'
+
+/**
+ * The journal entries of the engine's commits on a ticket: the leftover
+ * scan, dropped changesets, and the commit itself.
+ */
+
+/**
+ * The commits the engine makes on a ticket: tests first (`red`), then code
+ * (`green`), then one per review fix round (`fix`).
+ */
+export const CommitStageSchema = z.enum(['red', 'green', 'fix'])
+
+export type CommitStage = z.infer<typeof CommitStageSchema>
+
+/**
+ * The leftover scan before an engine commit. Any hit blocks the commit.
+ * `passed` lists the new markdown it let through and why (#520); left out
+ * when there was none, and in journals from before it.
+ */
+export const LeftoverScanEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('leftover_scan'),
+    content: z.object({
+        stage: CommitStageSchema,
+        hits: z.array(LeftoverHitSchema),
+        passed: z.array(LeftoverPassSchema).optional(),
+    }),
+})
+
+/**
+ * Before an engine commit's leftover scan, in a repo with changesets, the
+ * engine removed the changesets an agent wrote (new `.changeset/*.md` files
+ * other than the README) from the worktree: it writes the run's one itself.
+ * No ticket for the final review's fix commit.
+ */
+export const ChangesetDroppedEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('changeset_dropped'),
+    content: z.object({ paths: z.array(z.string().min(1)).min(1) }),
+})
+
+export const CommitMadeEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('commit_made'),
+    content: z.object({
+        stage: CommitStageSchema,
+        sha: z.string().min(1),
+        message: z.string(),
+        files: z.array(z.string()),
+    }),
+})

package/engine/journal/cross-ticket-replay.ts

@@ -0,0 +1,222 @@
+/**
+ * Records that reach across tickets in replay: a restarted engine's
+ * `run_resumed`, a closed or gone session, and the other joins an undo took
+ * off the run branch.
+ */
+import mapValues from 'lodash/mapValues'
+import omit from 'lodash/omit'
+import omitBy from 'lodash/omitBy'
+import uniq from 'lodash/uniq'
+
+import { isFixerRole } from './agent-turn-replay'
+import type { JournalRecord } from './journal-record'
+import type { RunState } from './run-state'
+import { EMPTY_TICKET_PROGRESS } from './ticket-progress'
+
+import { AgentRoleSchema, type AgentRole } from '../agents/role-results'
+
+/** A failed turn's session forgotten, when it is this role's. */
+const failureWithoutSession = <
+    Failure extends { role: AgentRole; session_id: string | null },
+>({
+    failure,
+    role,
+}: {
+    failure: Failure | null | undefined
+    role: AgentRole
+}): Failure | null =>
+    failure === null || failure === undefined
+        ? null
+        : failure.role === role
+          ? { ...failure, session_id: null }
+          : failure
+
+/**
+ * A restarted engine's `run_resumed`: each agent turn a crash cut off is
+ * taken again in a fresh session, so its role's session (and a failed
+ * turn's session, for a follow-up) is forgotten, on its ticket or in the
+ * final review, and the fresh agent is told of the crash. The counts it
+ * adds live in `crashes`.
+ */
+export const resumedAfter = ({
+    state,
+    record,
+}: {
+    state: RunState
+    record: Extract<JournalRecord, { kind: 'run_resumed' }>
+}): RunState =>
+    record.content.interrupted.reduce((current, { key, ticket, role }) => {
+        const parsed = AgentRoleSchema.safeParse(role)
+        if (!parsed.success) return current
+        const agent = parsed.data
+        if (key === 'final') {
+            const review = current.final_review
+            const failed = failureWithoutSession({
+                failure: review.agent_failures[agent],
+                role: agent,
+            })
+            return {
+                ...current,
+                final_review: {
+                    ...review,
+                    sessions: omit(review.sessions, agent),
+                    agent_failures:
+                        failed === null
+                            ? review.agent_failures
+                            : { ...review.agent_failures, [agent]: failed },
+                    crashed_turn: isFixerRole(agent)
+                        ? agent
+                        : review.crashed_turn,
+                },
+            }
+        }
+        if (ticket === null || key !== String(ticket)) return current
+        const progress = current.tickets[ticket] ?? EMPTY_TICKET_PROGRESS
+        return {
+            ...current,
+            tickets: {
+                ...current.tickets,
+                [ticket]: {
+                    ...progress,
+                    sessions: omit(progress.sessions, agent),
+                    agent_failure: failureWithoutSession({
+                        failure: progress.agent_failure,
+                        role: agent,
+                    }),
+                    crashed_turn: agent,
+                },
+            },
+        }
+    }, state)
+
+/**
+ * A closed session can take no follow-up: it is forgotten wherever the
+ * decision step would follow it up (a role's latest session, or a failed
+ * turn's), on its ticket, in the final review, or the learner's, so a fix
+ * round that would have gone to it goes to a fresh agent.
+ */
+export const closedSessionAfter = ({
+    state,
+    session_id,
+}: {
+    state: RunState
+    session_id: string
+}): RunState => {
+    const forget = <Failure extends { session_id: string | null }>(
+        failure: Failure
+    ): Failure =>
+        failure.session_id === session_id
+            ? { ...failure, session_id: null }
+            : failure
+    const withoutIt = (sessions: Partial<Record<AgentRole, string>>) =>
+        omitBy(sessions, (id) => id === session_id)
+    const review = state.final_review
+    const { learner } = state.memory
+    return {
+        ...state,
+        tickets: mapValues(state.tickets, (progress) => ({
+            ...progress,
+            sessions: withoutIt(progress.sessions),
+            agent_failure:
+                progress.agent_failure === null
+                    ? null
+                    : forget(progress.agent_failure),
+        })),
+        final_review: {
+            ...review,
+            sessions: withoutIt(review.sessions),
+            agent_failures: mapValues(review.agent_failures, (failure) =>
+                failure === undefined ? failure : forget(failure)
+            ),
+        },
+        memory: {
+            ...state.memory,
+            learner: {
+                ...learner,
+                agent_failure:
+                    learner.agent_failure === null
+                        ? null
+                        : forget(learner.agent_failure),
+            },
+        },
+    }
+}
+
+/**
+ * The session a follow-up found gone (#521), or `null`: no agent ran, so the
+ * record changes no step and counts no engine failure.
+ */
+export const goneSessionOf = (record: JournalRecord): string | null =>
+    record.kind === 'agent_failed' && record.content.session_gone
+        ? record.content.session_id
+        : null
+
+/**
+ * Closed sessions after a record: one journaled as closed, or found gone by
+ * a follow-up (#521), joins them and is forgotten; a later turn's end that
+ * names a closed session again (`agent_finished`, `agent_failed`) has it
+ * forgotten again, so it is never a follow-up target.
+ */
+export const closedSessionsAfter = ({
+    state,
+    record,
+}: {
+    state: RunState
+    record: JournalRecord
+}): RunState => {
+    const closed =
+        record.kind === 'agent_session_closed'
+            ? record.content.session_id
+            : goneSessionOf(record)
+    if (closed !== null) {
+        return closedSessionAfter({
+            state: {
+                ...state,
+                closed_sessions: uniq([...state.closed_sessions, closed]),
+            },
+            session_id: closed,
+        })
+    }
+    if (record.kind !== 'agent_finished' && record.kind !== 'agent_failed') {
+        return state
+    }
+    const { session_id } = record.content
+    return session_id !== null && state.closed_sessions.includes(session_id)
+        ? closedSessionAfter({ state, session_id })
+        : state
+}
+
+/**
+ * The other tickets whose joins an undo took off the run branch (#519): an
+ * undo resets the run branch to before the undone ticket's join, so any
+ * join on top of it goes too. Each such ticket's join and join gates are
+ * cleared, as its own undo would, so it joins again on the run branch as it
+ * is now, and a stuck one is never undone a second time. Its push, stuck,
+ * and reply are kept.
+ */
+export const undoneJoinsAfter = ({
+    state,
+    ticket,
+    shas,
+}: {
+    state: RunState
+    /** The ticket whose join was undone. */
+    ticket: number | null
+    /** Every commit the undo took off the run branch. */
+    shas: string[]
+}): RunState => {
+    if (shas.length === 0) return state
+    return {
+        ...state,
+        tickets: mapValues(state.tickets, (progress, number) => {
+            const { joined } = progress
+            const taken =
+                Number(number) !== ticket &&
+                joined?.ok === true &&
+                joined.shas.some((sha) => shas.includes(sha))
+            return taken
+                ? { ...progress, joined: null, join_gates: null }
+                : progress
+        }),
+    }
+}

package/engine/journal/decision-model-entries.ts

@@ -0,0 +1,86 @@
+import { z } from 'zod'
+
+import { ENTRY_FIELDS } from './entry-fields'
+
+import {
+    DecisionModelOffReasonSchema,
+    JevAnswerSchema,
+    JevFailureReasonSchema,
+    JevFixedSchema,
+    JevJobSchema,
+    JevRequestSchema,
+} from '../jev/jev-schemas'
+
+/**
+ * The journal entries of the decision model in shadow mode: each ask, its
+ * answers or failure, and the model turned off for the run.
+ */
+
+/**
+ * The engine asked the decision model (Clef; Jev in older journals), in
+ * shadow mode, with its own fixed choice beside. `model` is the model asked;
+ * journals from before #534 have none. `acted` is true when the answer can
+ * change what the engine does, false for a shadow ask; older journals read
+ * it as false.
+ */
+export const JevAskedEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('jev_asked'),
+    content: z.object({
+        job: JevJobSchema,
+        request: JevRequestSchema,
+        fixed: JevFixedSchema,
+        model: z.string().optional(),
+        acted: z.boolean().default(false),
+    }),
+})
+
+/**
+ * The decision model's answers to the `jev_asked` record at `asked_seq`.
+ * The engine ignores them. `model` is the model that answered, as it named
+ * itself (such as `clef`); journals from before #534 have none.
+ */
+export const JevAnsweredEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('jev_answered'),
+    content: z.object({
+        job: JevJobSchema,
+        asked_seq: z.number().int().min(1),
+        answers: z.record(z.string(), JevAnswerSchema),
+        /** How long the call took. */
+        ms: z.number().min(0),
+        model: z.string().optional(),
+    }),
+})
+
+/** The decision model call at `asked_seq` gave no answers. The run goes on regardless. */
+export const JevFailedEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('jev_failed'),
+    content: z.object({
+        job: JevJobSchema,
+        asked_seq: z.number().int().min(1),
+        reason: JevFailureReasonSchema,
+        error: z.string(),
+        ms: z.number().min(0),
+    }),
+})
+
+/**
+ * The decision model is off for the rest of the run (#534), so the engine
+ * asks it nothing more: `no_credentials` at the start (Luca's env file and
+ * the process env have no Cloudflare account id or token), or `rejected`
+ * after an ask Cloudflare turned down. `model` is the model it would have
+ * asked; `detail` is one plain sentence (the file and keys to add, or
+ * Cloudflare's answer), never the token. Shadow mode only: it changes
+ * nothing in the run.
+ */
+export const DecisionModelOffEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('decision_model_off'),
+    content: z.object({
+        model: z.string(),
+        reason: DecisionModelOffReasonSchema,
+        detail: z.string(),
+    }),
+})

package/engine/journal/entry-fields.ts

@@ -0,0 +1,10 @@
+import { z } from 'zod'
+
+/** The fields every journal entry has, whatever its kind. */
+
+export const ENTRY_FIELDS = {
+    /** The ticket (or spec) a record is about, `null` for the whole run. */
+    ticket: z.number().int().positive().nullable(),
+    /** The role that produced the record, `null` for the engine itself. */
+    role: z.string().nullable(),
+}