@alecsibilia/luca 14.0.0-alpha.21 → 14.0.0-alpha.22

package/engine/testing/effort-pick-fixtures.ts

@@ -0,0 +1,139 @@
+import {
+    runOf,
+    ticketRun,
+    type Built,
+    type Effort,
+    type Severity,
+} from './model-score-fixtures'
+
+import { ModelsConfigSchema } from '../config/engine-config'
+import type { JournalEntry, JournalRecord } from '../journal/journal-record'
+import type { ModelReport } from '../models/model-score'
+
+/**
+ * Journal builders for the score report's "Clef's effort picks" section
+ * (#663): Clef's shadow-mode `agent_effort` answers, the tickets they were
+ * asked on, and a run with its own confidence bar.
+ */
+
+/** Clef's `agent_effort` answer about `role`. */
+export type EffortPick = {
+    role: Built
+    pick: Effort
+    confidence: number | null
+}
+
+/**
+ * An `agent_effort` ask about `role`, in shadow mode, and Clef's answer.
+ * `runOf` points the answer's `asked_seq` at the ask.
+ */
+export const effortAsk = ({
+    ticket,
+    role,
+    pick,
+    confidence,
+}: EffortPick & { ticket: number }): JournalEntry[] => [
+    {
+        kind: 'jev_asked',
+        ticket,
+        role,
+        content: {
+            job: 'agent_effort',
+            request: {
+                state: {},
+                questions: {
+                    effort: {
+                        type: 'choice',
+                        instructions:
+                            'Which effort should this agent use? Pick the lowest that will get it right.',
+                        criteria: { medium: null, high: null, xhigh: null },
+                    },
+                },
+            },
+            fixed: { effort: 'medium' },
+            acted: false,
+        },
+    },
+    {
+        kind: 'jev_answered',
+        ticket,
+        role,
+        content: {
+            job: 'agent_effort',
+            asked_seq: 1,
+            answers: { effort: { value: pick, confidence, raw: {} } },
+            ms: 5,
+        },
+    },
+]
+
+/**
+ * One ticket after Clef's effort `picks`: the test-writer at `tw_effort`
+ * and the implementer at `impl_effort` (both medium unless told), every
+ * ticket review at xhigh, the first review's `findings`, 100 lines.
+ */
+export const pickedTicket = ({
+    ticket,
+    tw_effort = 'medium',
+    impl_effort = 'medium',
+    picks = [],
+    findings = [],
+}: {
+    ticket: number
+    tw_effort?: Effort
+    impl_effort?: Effort
+    picks?: EffortPick[]
+    findings?: Severity[]
+}): JournalEntry[] => [
+    ...picks.flatMap((pick) => effortAsk({ ticket, ...pick })),
+    ...ticketRun({
+        ticket,
+        tw_effort,
+        impl_effort,
+        review_effort: 'xhigh',
+        findings,
+    }),
+]
+
+/** A run of `body` whose config sets the confidence bar to `bar`. */
+export const runWithBar = (
+    body: JournalEntry[],
+    bar: number
+): JournalRecord[] =>
+    runOf(body).map((record) =>
+        record.kind === 'run_started'
+            ? {
+                  ...record,
+                  content: {
+                      ...record.content,
+                      config: {
+                          ...record.content.config,
+                          models: ModelsConfigSchema.parse({
+                              confidence_bar: bar,
+                          }),
+                      },
+                  },
+              }
+            : record
+    )
+
+/** The report's "Clef's effort picks" for `role`. */
+export const effortPicksOf = (report: ModelReport, role: Built) =>
+    report.effort_picks.find((picks) => picks.role === role)
+
+/**
+ * The lines of `role`'s "Clef's effort picks (shadow mode)" section in the
+ * printed report: the lines after its heading, within the role's part.
+ */
+export const pickSection = (text: string, role: Built): string[] => {
+    const lines = text.split('\n')
+    const start = lines.indexOf(role)
+    const end = lines.findIndex(
+        (line, at) => at > start && line !== '' && !line.startsWith(' ')
+    )
+    const part = start < 0 ? [] : lines.slice(start + 1, end)
+    const heading = part.findIndex(
+        (line) => line.trim() === "Clef's effort picks (shadow mode)"
+    )
+    return heading < 0 ? [] : part.slice(heading + 1)
+}

package/engine/testing/model-score-fixtures.ts

@@ -0,0 +1,300 @@
+import {
+    agentFailed,
+    agentSession,
+    finding,
+    implemented,
+    intakePassed,
+    practiceTicket,
+    pushed,
+    reviewed,
+    testsWritten,
+} from './build-fixtures'
+import { recordsFrom } from './intake-fixtures'
+
+import type { AgentRole } from '../agents/role-results'
+import type { JournalEntry, JournalRecord } from '../journal/journal-record'
+import type { ModelScore } from '../models/model-score'
+
+/**
+ * Journal builders for the model score report's tests (#605): a ticket
+ * built with each role on its model and effort, a run's records, and
+ * lookups into the report's scores.
+ */
+
+export type Model = 'opus' | 'sonnet' | 'haiku'
+export type Effort = 'low' | 'medium' | 'high' | 'xhigh' | 'max'
+export type Severity = 'blocker' | 'should_fix' | 'nit'
+export type Built = 'test-writer' | 'implementer'
+
+/**
+ * An agent turn started on `model`; `null` writes one from before #600. It
+ * ran at `effort`; left out writes one from before #659, with no effort.
+ */
+export const started = ({
+    ticket,
+    role,
+    model,
+    follow_up_of,
+    effort,
+}: {
+    ticket: number | null
+    role: AgentRole
+    model: Model | null
+    follow_up_of?: string
+    effort?: Effort
+}): JournalEntry => ({
+    kind: 'agent_started',
+    ticket,
+    role,
+    content: {
+        role,
+        prompt: 'p',
+        follow_up_of: follow_up_of ?? null,
+        ...(model === null ? {} : { model }),
+        ...(effort === undefined ? {} : { effort }),
+    },
+})
+
+/**
+ * An `agent_model` ask about `role` and Clef's answer, or with `role` null
+ * an older journal's `ticket_model` ask, which named full model ids.
+ * `runOf` points the answer's `asked_seq` at the ask.
+ */
+export const modelAsk = ({
+    ticket,
+    role,
+    pick,
+    confidence,
+    acted,
+}: {
+    ticket: number | null
+    role: Built | 'learner' | null
+    pick: string
+    confidence: number | null
+    acted?: boolean
+}): JournalEntry[] => {
+    const job = role === null ? 'ticket_model' : 'agent_model'
+    const options =
+        role === null
+            ? ['claude-opus-5-5', 'claude-sonnet-5', 'claude-haiku-4-5']
+            : ['opus', 'sonnet', 'haiku']
+    return [
+        {
+            kind: 'jev_asked',
+            ticket,
+            role,
+            content: {
+                job,
+                request: {
+                    state: {},
+                    questions: {
+                        model: {
+                            type: 'choice',
+                            instructions: 'Which model?',
+                            criteria: Object.fromEntries(
+                                options.map((option) => [option, null])
+                            ),
+                        },
+                    },
+                },
+                fixed: { model: options[0] ?? 'opus' },
+                ...(acted === undefined ? {} : { acted }),
+            },
+        },
+        {
+            kind: 'jev_answered',
+            ticket,
+            role,
+            content: {
+                job,
+                asked_seq: 1,
+                answers: { model: { value: pick, confidence, raw: {} } },
+                ms: 5,
+            },
+        },
+    ]
+}
+
+export type Ask = Omit<Parameters<typeof modelAsk>[0], 'ticket'>
+
+/**
+ * One ticket built: the test-writer on `tw`, the implementer on `impl`
+ * (opus unless told), the first review's `findings`, then `rounds` fix
+ * rounds (one when the findings block), each fix's review finding a
+ * should-fix until the last approves, and a join of `lines` lines. Each
+ * role's agents run at its effort, left out as before #659.
+ */
+export const ticketRun = ({
+    ticket,
+    tw = 'opus',
+    impl = 'opus',
+    tw_effort,
+    impl_effort,
+    review_effort,
+    asks = [],
+    findings = [],
+    rounds,
+    lines = 100,
+    tokens = 100,
+    tw_failed_first = false,
+}: {
+    ticket: number
+    tw?: Model | null
+    impl?: Model | null
+    tw_effort?: Effort
+    impl_effort?: Effort
+    review_effort?: Effort
+    asks?: Ask[]
+    findings?: Severity[]
+    rounds?: number
+    /** `null` joins as an older journal does, with no `lines_changed`. */
+    lines?: number | null
+    /** The ticket's tokens, all in its test-writer's session. */
+    tokens?: number
+    /** The test-writer on `tw` fails a try; a fresh one on opus follows. */
+    tw_failed_first?: boolean
+}): JournalEntry[] => {
+    const askFor = (role: Built | null) =>
+        asks
+            .filter((ask) => ask.role === role)
+            .flatMap((ask) => modelAsk({ ticket, ...ask }))
+    const fixes =
+        rounds ?? (findings.some((severity) => severity !== 'nit') ? 1 : 0)
+    const review = (list: Severity[], index: number): JournalEntry[] => [
+        started({
+            ticket,
+            role: 'ticket-reviewer',
+            model: 'opus',
+            effort: review_effort,
+        }),
+        reviewed({
+            ticket,
+            findings: list.map((severity, at) =>
+                finding({ id: `F${index}-${at}`, severity })
+            ),
+        }),
+    ]
+    return [
+        ...askFor(null),
+        ...askFor('test-writer'),
+        ...(tw_failed_first
+            ? [
+                  started({
+                      ticket,
+                      role: 'test-writer',
+                      model: tw,
+                      effort: tw_effort,
+                  }),
+                  agentFailed({
+                      ticket,
+                      role: 'test-writer',
+                      failure: 'agent',
+                  }),
+              ]
+            : []),
+        started({
+            ticket,
+            role: 'test-writer',
+            model: tw_failed_first ? 'opus' : tw,
+            effort: tw_effort,
+        }),
+        testsWritten({ ticket }),
+        agentSession({
+            ticket,
+            role: 'test-writer',
+            output_tokens: tokens - 10,
+        }),
+        ...askFor('implementer'),
+        started({
+            ticket,
+            role: 'implementer',
+            model: impl,
+            effort: impl_effort,
+        }),
+        implemented({ ticket }),
+        ...review(findings, 0),
+        ...Array.from({ length: fixes }, (_, index) => [
+            started({
+                ticket,
+                role: 'implementer',
+                model: impl,
+                follow_up_of: 'impl-1',
+                effort: impl_effort,
+            }),
+            implemented({ ticket }),
+            ...review(index + 1 < fixes ? ['should_fix'] : [], index + 1),
+        ]).flat(),
+        {
+            kind: 'ticket_joined',
+            ticket,
+            role: null,
+            content: {
+                ok: true,
+                shas: ['r1', 'g1'],
+                ...(lines === null ? {} : { lines_changed: lines }),
+            },
+        },
+        pushed({ ticket }),
+        {
+            kind: 'usage_recorded',
+            ticket,
+            role: null,
+            content: {
+                scope: 'ticket',
+                ticket,
+                agent_turns: 1,
+                tokens: {
+                    input_tokens: 10,
+                    output_tokens: tokens - 10,
+                    cache_read_input_tokens: 0,
+                    cache_creation_input_tokens: 0,
+                },
+                windows: {},
+            },
+        },
+    ]
+}
+
+/** A run's records: its intake, then `body`, each answer after its ask. */
+export const runOf = (body: JournalEntry[]): JournalRecord[] => {
+    const numbers = [
+        ...new Set(
+            body.flatMap(({ ticket }) => (ticket === null ? [] : [ticket]))
+        ),
+    ]
+    const entries = [
+        ...intakePassed({
+            tickets: numbers.map((number) => practiceTicket({ number })),
+        }),
+        ...body,
+    ]
+    return recordsFrom({
+        entries: entries.map((entry, index) =>
+            entry.kind === 'jev_answered'
+                ? { ...entry, content: { ...entry.content, asked_seq: index } }
+                : entry
+        ),
+    })
+}
+
+export const scoreOf = (
+    scores: ModelScore[],
+    role: Built,
+    model: Model
+): ModelScore | undefined =>
+    scores.find((score) => score.role === role && score.model === model)
+
+/** The role and model of each score with tickets, sorted. */
+export const groups = (scores: ModelScore[]): string[] =>
+    scores
+        .filter(({ tickets }) => tickets > 0)
+        .map(({ role, model }) => `${role}/${model}`)
+        .sort()
+
+/** Clef picked `pick` for the test-writer, sure enough to act on it. */
+export const sure = (pick: Model, confidence = 0.9): Ask => ({
+    role: 'test-writer',
+    pick,
+    confidence,
+    acted: true,
+})

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@alecsibilia/luca",
     "description": "Luca: turns a spec on GitHub Issues into a reviewed pull request, run from Paseo with Claude Code",
-    "version": "14.0.0-alpha.21",
+    "version": "14.0.0-alpha.22",
     "license": "Apache-2.0",
     "author": "Alec Sibilia <sibilia.alec@gmail.com>",
     "homepage": "https://github.com/asibilia/luca-framework#readme",