@alecsibilia/luca 14.0.0-alpha.27 → 14.0.0-alpha.28

package/CHANGELOG.md

@@ -1,5 +1,9 @@
 # @alecsibilia/luca
 
+## 14.0.0-alpha.28
+
+No changes in this release.
+
 ## 14.0.0-alpha.27
 
 No changes in this release.

package/board/client/model-line.tsx

@@ -36,8 +36,8 @@ const capital = ({ text }: { text: string }): string =>
  * doesn't know.
  *
  * @example
- * clefLine({ model: { role: 'implementer', name: 'opus', reason: 'unsure', answer: { role: 'implementer', pick: 'sonnet', confidence: 0.56 } }, bar: 0.8 })
- * // { short: '0.56 split → stayed on opus', full: 'Clef picked sonnet but was only 0.56 sure (split: under 0.8), so Luca stayed on opus.', confidence: 0.56, sharp: false }
+ * clefLine({ model: { role: 'implementer', name: 'opus', reason: 'unsure', answer: { role: 'implementer', pick: 'sonnet', confidence: 0.36 } }, bar: 0.5 })
+ * // { short: '0.36 split → stayed on opus', full: 'Clef picked sonnet but was only 0.36 sure (split: under 0.5), so Luca stayed on opus.', confidence: 0.36, sharp: false }
  */
 export const clefLine = ({
     model,
@@ -54,10 +54,11 @@ export const clefLine = ({
     const sure = confidence?.toFixed(2) ?? ''
     const judged = `${word}: ${sharp ? 'at or over' : 'under'} ${bar}`
     const plain = { confidence: null, sharp: false }
+    const picked = `${confidence === null ? 'clef' : `${sure} ${word}`} → ${name}`
     switch (reason) {
         case 'decision_model':
             return {
-                short: `${confidence === null ? 'clef' : `${sure} ${word}`} → ${name}`,
+                short: picked,
                 full:
                     confidence === null
                         ? `Clef picked ${name}, so Luca ran the ${role} on it.`
@@ -65,6 +66,16 @@ export const clefLine = ({
                 confidence,
                 sharp,
             }
+        case 'trial':
+            return {
+                short: `${picked} (trial)`,
+                full:
+                    confidence === null
+                        ? `Clef picked ${name}, and Luca ran the ${role} on it in a model trial.`
+                        : `Clef picked ${name} and was ${sure} sure (${judged}), and Luca ran the ${role} on it in a model trial.`,
+                confidence,
+                sharp,
+            }
         case 'unsure':
             return {
                 short: `${confidence === null ? 'unsure' : `${sure} ${word}`} → stayed on ${name}`,

package/board/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@luca/board",
-    "version": "14.0.0-alpha.27",
+    "version": "14.0.0-alpha.28",
     "private": true,
     "type": "module"
 }

package/board/server/luca-version.ts

@@ -1,2 +1,2 @@
 /** The Luca version of this board, stamped when the package was packed. */
-export const LUCA_VERSION: string = "14.0.0-alpha.27"
+export const LUCA_VERSION: string = "14.0.0-alpha.28"

package/board/shared/board-state-run.ts

@@ -94,7 +94,7 @@ export const DEFAULT_RUN_BUDGET_TOKENS = 9_000_000
  * `engine/src/config/engine-config-schema.ts`): the decision model is sharp at or
  * over it, split under it. Kept here for the same reason.
  */
-export const DEFAULT_CONFIDENCE_BAR = 0.8
+export const DEFAULT_CONFIDENCE_BAR = 0.5
 
 export const EngineEndedSchema = z.object({
     ok: z.boolean(),

package/engine/config/engine-config-schema.ts

@@ -216,12 +216,15 @@ const RoleEffortsSchema = z.strictObject(
 /**
  * The config's `models` block (#603): each routed role's model floor, each
  * writer's effort (#660), and how sure the decision model must be
- * (`confidence_bar`) before its model pick is used.
+ * (`confidence_bar`) before its model pick is used, and the share of its
+ * unsure but cheaper picks that run anyway in a model trial (`trial_share`,
+ * #698).
  */
 export const ModelsConfigSchema = z.object({
     floors: ModelFloorsSchema.prefault({}),
     effort: RoleEffortsSchema.prefault({}),
-    confidence_bar: z.number().min(0).max(1).default(0.8),
+    confidence_bar: z.number().min(0).max(1).default(0.5),
+    trial_share: z.number().min(0).max(1).default(0),
 })
 
 export type ModelsConfig = z.infer<typeof ModelsConfigSchema>

package/engine/core/agent-turn.ts

@@ -3,6 +3,8 @@
  * ended, journaled. Also the `launch_agent` and `follow_up_agent` steps,
  * which start a turn.
  */
+import { basename } from 'node:path'
+
 import { need, ticketWorktree, type BuildContext } from './build-context'
 import { mayEditTests, type BuildAction } from './decide-build'
 import { doChores } from './engine-chores'
@@ -19,7 +21,11 @@ import { DEFAULT_MODEL, type EffortLevel } from '../config/engine-config'
 import { describeViolations } from '../guards/after-turn-check'
 import { guardRoleOf } from '../guards/role-rules'
 import { enforceAfterTurn, snapshotWorktree } from '../guards/worktree-state'
-import { launchPick, type ModelReason } from '../jev/model-pick'
+import {
+    launchPick,
+    type ModelReason,
+    type TrialGroup,
+} from '../jev/model-pick'
 import type { AgentFailure, JournalRecord } from '../journal/journal-record'
 import type { ReplayedWorktree } from '../journal/replay'
 import { sessionSignal } from '../limits/plan-signals'
@@ -253,13 +259,15 @@ const sessionOf = (record: JournalRecord): string | null => {
 }
 
 /**
- * The model a session launched on, and why (`null` before #603), and the
- * effort it runs at (`null` before #659).
+ * The model a session launched on, and why (`null` before #603), the
+ * effort it runs at (`null` before #659), and its model trial group
+ * (`null` when not eligible, or before #698).
  */
 export type SessionModel = {
     model: string
     model_reason: ModelReason | null
     effort: EffortLevel | null
+    trial_group: TrialGroup | null
 }
 
 /**
@@ -302,8 +310,8 @@ export const sessionModel = ({
                           record.content.follow_up_of === null
                   ))
     if (started?.kind !== 'agent_started') return null
-    const { model, model_reason, effort } = started.content
-    return model === null ? null : { model, model_reason, effort }
+    const { model, model_reason, effort, trial_group } = started.content
+    return model === null ? null : { model, model_reason, effort, trial_group }
 }
 
 /**
@@ -321,6 +329,7 @@ export const followUpModel = ({
         model: DEFAULT_MODEL,
         model_reason: null,
         effort: null,
+        trial_group: null,
     }
 
 export const launchAgent = async ({
@@ -335,10 +344,11 @@ export const launchAgent = async ({
     // a routed role's model; a reviewer is locked on Opus. A redo keeps
     // the model its first try journaled. The role's effort applies on any
     // model (#659).
-    const { model, model_reason } = launchPick({
+    const { model, model_reason, trial_group } = launchPick({
         records: context.journal.read(),
         state: context.state,
         config: context.config,
+        run_id: basename(context.run_dir),
         ticket,
         role,
         step: context.step,
@@ -355,6 +365,7 @@ export const launchAgent = async ({
             model,
             model_reason,
             effort,
+            trial_group,
         },
     })
     await runTurn({
@@ -394,7 +405,7 @@ export const followUpAgent = async ({
         value: context.state.snapshot?.tickets[ticket],
         what: `snapshot of #${ticket}`,
     })
-    const { model, model_reason, effort } = followUpModel({
+    const { model, model_reason, effort, trial_group } = followUpModel({
         records: context.journal.read(),
         session_id,
     })
@@ -409,6 +420,7 @@ export const followUpAgent = async ({
             model,
             model_reason,
             effort,
+            trial_group,
         },
     })
     await runTurn({

package/engine/core/execute-final-review.ts

@@ -265,6 +265,7 @@ export const executeFinalReviewAction = async ({
                     model,
                     model_reason,
                     effort,
+                    trial_group: null,
                 },
             })
             await runTurn({

package/engine/core/execute-memory.ts

@@ -1,3 +1,5 @@
+import { basename } from 'node:path'
+
 import type { MemoryAction, MemorySaveStep } from './decide-memory'
 import { buildContext, need, runTurn, type BuildDeps } from './execute-build'
 
@@ -413,6 +415,7 @@ export const executeMemoryAction = async ({
                 records: journal.read(),
                 state: built.state,
                 config: built.config,
+                run_id: basename(built.run_dir),
                 ticket: null,
                 role: 'learner',
                 step,
@@ -432,6 +435,8 @@ export const executeMemoryAction = async ({
                     model,
                     model_reason,
                     effort,
+                    // The learner is never in a model trial.
+                    trial_group: null,
                 },
             })
             await runTurn({

package/engine/jev/model-pick.ts

@@ -29,6 +29,8 @@ import type { RunState } from '../journal/replay'
  * - `unsure`: its confidence was below the bar or missing, or it picked a
  *   model the ask didn't offer.
  * - `decision_model`: the decision model's pick.
+ * - `trial`: an unsure but cheaper pick that runs anyway, in a model
+ *   trial's trial group (#698).
  */
 export const ModelReasonSchema = z.enum([
     'locked',
@@ -37,19 +39,79 @@ export const ModelReasonSchema = z.enum([
     'no_answer',
     'unsure',
     'decision_model',
+    'trial',
 ])
 
 export type ModelReason = z.infer<typeof ModelReasonSchema>
 
-/** The model an agent launches on, and why. */
-export type ModelPick = { model: ModelName; model_reason: ModelReason }
+/**
+ * A model trial's group (#698): `trial` runs Clef's unsure but cheaper
+ * pick, `control` stays on Opus.
+ */
+export const TrialGroupSchema = z.enum(['trial', 'control'])
+
+export type TrialGroup = z.infer<typeof TrialGroupSchema>
+
+/**
+ * The model an agent launches on, why, and its model trial group (`null`
+ * when the launch isn't eligible for a trial).
+ */
+export type ModelPick = {
+    model: ModelName
+    model_reason: ModelReason
+    trial_group: TrialGroup | null
+}
 
 /** Reviewers, lenses, and the final review's fixers: always Opus. */
 export const LOCKED_PICK: ModelPick = {
     model: DEFAULT_MODEL,
     model_reason: 'locked',
+    trial_group: null,
 }
 
+/**
+ * A launch's model trial group (#698), the same for the same key every
+ * time: the first 8 hex digits of the key's SHA-256, over `0x100000000`,
+ * below `share` is `trial`, else `control`.
+ *
+ * @example
+ * trialGroupOf({ key: 'luca-20261028-202119-k6q1:697:test-writer', share: 0.5 }) // 'trial'
+ */
+export const trialGroupOf = ({
+    key,
+    share,
+}: {
+    key: string
+    share: number
+}): TrialGroup => {
+    const hex = new Bun.CryptoHasher('sha256').update(key).digest('hex')
+    return parseInt(hex.slice(0, 8), 16) / 0x100000000 < share
+        ? 'trial'
+        : 'control'
+}
+
+/**
+ * The text a launch's model trial group is hashed from: the run folder's
+ * name, the ticket's bare number and the role; `null` for a launch that
+ * can't be in a trial (the learner, or one with no ticket).
+ *
+ * @example
+ * trialKeyOf({ run_id: 'luca-20261028-202119-k6q1', ticket: 697, role: 'test-writer' })
+ * // 'luca-20261028-202119-k6q1:697:test-writer'
+ */
+export const trialKeyOf = ({
+    run_id,
+    ticket,
+    role,
+}: {
+    run_id: string
+    ticket: number | null
+    role: AgentRole
+}): string | null =>
+    ticket === null || (role !== 'test-writer' && role !== 'implementer')
+        ? null
+        : `${run_id}:${ticket}:${role}`
+
 export type RoutedRole = (typeof ROUTED_ROLES)[number]
 
 /** Whether a role has a model floor, so the decision model may pick its model. */
@@ -84,38 +146,69 @@ export type ModelAnswer = {
  * offered is `unsure`; else the decision model's pick. Every fallback is
  * Opus.
  *
+ * An `unsure` pick is in a model trial (#698) when `trial_share` is above
+ * 0, the launch has a `trial_key`, its only fault is a confidence below
+ * the bar, and it picked an offered model at or above the floor and
+ * cheaper than Opus: its trial group runs the pick as `trial`, its control
+ * group stays on Opus as `unsure`.
+ *
  * @example
- * pickModel({ role: 'test-writer', models, failed_cheaper: false, answer: { offered: ['sonnet', 'opus'], pick: 'sonnet', confidence: 0.9 } })
- * // { model: 'sonnet', model_reason: 'decision_model' } with a sonnet floor and the 0.8 bar
+ * pickModel({ role: 'test-writer', models, failed_cheaper: false, answer: { offered: ['sonnet', 'opus'], pick: 'sonnet', confidence: 0.9 }, trial_key: null })
+ * // { model: 'sonnet', model_reason: 'decision_model', trial_group: null } with a sonnet floor and the 0.5 bar
  */
 export const pickModel = ({
     role,
     models,
     failed_cheaper,
     answer,
+    trial_key,
 }: {
     role: AgentRole
     models: ModelsConfig
     /** Whether the role already failed a try on its ticket on a cheaper model. */
     failed_cheaper: boolean
     answer: ModelAnswer | null
+    /** The text hashed for the trial group; `null` if it can't be in one. */
+    trial_key: string | null
 }): ModelPick => {
+    const opus = (model_reason: ModelReason): ModelPick => ({
+        model: 'opus',
+        model_reason,
+        trial_group: null,
+    })
     if (!isRoutedRole(role)) return LOCKED_PICK
-    if (failed_cheaper) return { model: 'opus', model_reason: 'after_failure' }
-    if (models.floors[role] === 'opus') {
-        return { model: 'opus', model_reason: 'floor' }
-    }
-    if (answer === null) return { model: 'opus', model_reason: 'no_answer' }
+    if (failed_cheaper) return opus('after_failure')
+    const floor = models.floors[role]
+    if (floor === 'opus') return opus('floor')
+    if (answer === null) return opus('no_answer')
     const pick = ModelNameSchema.safeParse(answer.pick)
     if (
         !pick.success ||
         !answer.offered.includes(pick.data) ||
-        answer.confidence === null ||
-        answer.confidence < models.confidence_bar
+        answer.confidence === null
+    ) {
+        return opus('unsure')
+    }
+    if (answer.confidence >= models.confidence_bar) {
+        return {
+            model: pick.data,
+            model_reason: 'decision_model',
+            trial_group: null,
+        }
+    }
+    const all = ModelNameSchema.options
+    if (
+        trial_key === null ||
+        models.trial_share <= 0 ||
+        pick.data === 'opus' ||
+        all.indexOf(pick.data) > all.indexOf(floor)
     ) {
-        return { model: 'opus', model_reason: 'unsure' }
+        return opus('unsure')
     }
-    return { model: pick.data, model_reason: 'decision_model' }
+    return trialGroupOf({ key: trial_key, share: models.trial_share }) ===
+        'trial'
+        ? { model: pick.data, model_reason: 'trial', trial_group: 'trial' }
+        : { ...opus('unsure'), trial_group: 'control' }
 }
 
 /**
@@ -211,9 +304,9 @@ export const redonePick = ({
     )
     if (started?.kind !== 'agent_started') return null
     const model = ModelNameSchema.safeParse(started.content.model)
-    const { model_reason } = started.content
+    const { model_reason, trial_group } = started.content
     if (!model.success || model_reason === null) return null
-    return { model: model.data, model_reason }
+    return { model: model.data, model_reason, trial_group }
 }
 
 /** Which try of its step a launch is: a redo names its first try's seq. */
@@ -246,13 +339,16 @@ export const asksModel = ({
 
 /**
  * The model a fresh agent launches on, from the run's config and the
- * answer the launch step just journaled. A redo launches on the model its
- * first try journaled.
+ * answer the launch step just journaled. A redo launches on the model, and
+ * in the trial group, its first try journaled.
+ *
+ * @param run_id - The run folder's name, for the model trial's key.
  */
 export const launchPick = ({
     records,
     state,
     config,
+    run_id,
     ticket,
     role,
     step,
@@ -260,6 +356,7 @@ export const launchPick = ({
     records: JournalRecord[]
     state: RunState
     config: EngineConfig
+    run_id: string
     ticket: number | null
     role: AgentRole
     step: LaunchTry
@@ -272,4 +369,5 @@ export const launchPick = ({
         models: modelsOf({ config }),
         failed_cheaper: failedCheaper({ state, ticket, role }),
         answer: modelAnswerOf({ records, ticket, role }),
+        trial_key: trialKeyOf({ run_id, ticket, role }),
     })

package/engine/journal/agent-entries.ts

@@ -10,7 +10,7 @@ import {
 } from '../agents/role-results'
 import { EffortLevelSchema } from '../config/engine-config'
 import { GateCheckSchema } from '../gates/gate-schemas'
-import { ModelReasonSchema } from '../jev/model-pick'
+import { ModelReasonSchema, TrialGroupSchema } from '../jev/model-pick'
 
 /**
  * The journal entries of agent turns: started, finished or failed, their
@@ -26,7 +26,9 @@ import { ModelReasonSchema } from '../jev/model-pick'
  * launched on that model (#603; a follow-up's is its session's); `null` in
  * journals from before it. `effort` is the effort it ran at (#659; a
  * follow-up's is its session's); `null` in journals from before it, when
- * every agent ran at `high`.
+ * every agent ran at `high`. `trial_group` is its model trial group (#698;
+ * a redo's is its first try's, a follow-up's is its session's); `null` when
+ * it wasn't eligible, and in journals from before it.
  */
 export const AgentStartedEntrySchema = z.object({
     ...ENTRY_FIELDS,
@@ -38,6 +40,7 @@ export const AgentStartedEntrySchema = z.object({
         model: z.string().nullable().default(null),
         model_reason: ModelReasonSchema.nullable().default(null),
         effort: EffortLevelSchema.nullable().default(null),
+        trial_group: TrialGroupSchema.nullable().default(null),
     }),
 })
 

package/engine/models/model-score-facts.ts

@@ -6,14 +6,13 @@ import {
 } from './model-score-rules'
 
 import type { FindingSeverity } from '../agents/role-results'
-import { modelsOf } from '../config/config-fields'
 import {
     EffortLevelSchema,
     ModelNameSchema,
-    ModelsConfigSchema,
     type EffortLevel,
     type ModelName,
 } from '../config/engine-config'
+import type { ModelReason, TrialGroup } from '../jev/model-pick'
 import type { JournalRecord } from '../journal/journal-record'
 import { countedTokens } from '../limits/plan-usage'
 
@@ -31,6 +30,9 @@ export type Pick = {
     shadow: boolean
 }
 
+/** Clef's effort pick for a role on a ticket, with its confidence. */
+export type EffortPick = { effort: EffortLevel; confidence: number }
+
 /** What one run's journal says about one ticket. */
 export type TicketFacts = {
     /** The model of each role's first agent. */
@@ -40,11 +42,18 @@ export type TicketFacts = {
     /** The first ticket review's effort, `high` when none is journaled. */
     review_effort: EffortLevel
     picks: Partial<Record<ScoredRole, Pick>>
+    /** Each role's first agent's model trial group, when it had one (#698). */
+    trial_groups: Partial<Record<ScoredRole, TrialGroup>>
+    /**
+     * Clef's first `agent_effort` answer for each role that has a
+     * confidence, at any confidence (#697).
+     */
+    effort_picks: Partial<Record<ScoredRole, EffortPick>>
     /**
-     * Clef's first `agent_effort` answer for each role at or above the run's
-     * confidence bar.
+     * Why each role's first agent got its model; `null` for one from before
+     * #603.
      */
-    effort_picks: Partial<Record<ScoredRole, EffortLevel>>
+    model_reasons: Partial<Record<ScoredRole, ModelReason | null>>
     reviews: number
     /** The first ticket review's weighted findings. */
     findings: number
@@ -100,7 +109,9 @@ const newTicket = (): TicketFacts => ({
     efforts: {},
     review_effort: EFFORT_BEFORE_659,
     picks: {},
+    trial_groups: {},
     effort_picks: {},
+    model_reasons: {},
     reviews: 0,
     findings: 0,
     fix_rounds: 0,
@@ -125,15 +136,11 @@ export const factsOf = (records: JournalRecord[]): RunFacts => {
     const effort_asks = new Map<number, string | null>()
     const reviewed = new Set<number>()
     const learner: LearnerRun[] = []
-    let bar = ModelsConfigSchema.parse({}).confidence_bar
     let routed = false
     let final_rounds = 0
     let final_findings = 0
     for (const record of records) {
         switch (record.kind) {
-            case 'run_started':
-                bar = modelsOf({ config: record.content.config }).confidence_bar
-                break
             case 'jev_asked':
                 if (record.content.job === 'agent_effort') {
                     effort_asks.set(record.seq, record.role)
@@ -159,11 +166,12 @@ export const factsOf = (records: JournalRecord[]): RunFacts => {
                     if (
                         isScored(effort_role) &&
                         level.success &&
-                        confidence !== null &&
-                        confidence >= bar
+                        confidence !== null
                     ) {
-                        ticketOf(record.ticket).effort_picks[effort_role] ??=
-                            level.data
+                        ticketOf(record.ticket).effort_picks[effort_role] ??= {
+                            effort: level.data,
+                            confidence,
+                        }
                     }
                     break
                 }
@@ -184,7 +192,14 @@ export const factsOf = (records: JournalRecord[]): RunFacts => {
                 break
             }
             case 'agent_started': {
-                const { role, model, effort, follow_up_of } = record.content
+                const {
+                    role,
+                    model,
+                    effort,
+                    follow_up_of,
+                    model_reason,
+                    trial_group,
+                } = record.content
                 const family = familyOf(model) ?? 'opus'
                 const level = effort ?? EFFORT_BEFORE_659
                 if (follow_up_of !== null) break
@@ -197,8 +212,14 @@ export const factsOf = (records: JournalRecord[]): RunFacts => {
                 if (record.ticket === null) break
                 if (isScored(role)) {
                     const facts = ticketOf(record.ticket)
-                    facts.models[role] ??= family
-                    facts.efforts[role] ??= level
+                    if (facts.models[role] === undefined) {
+                        facts.models[role] = family
+                        facts.efforts[role] = level
+                        facts.model_reasons[role] = model_reason
+                        if (trial_group !== null) {
+                            facts.trial_groups[role] = trial_group
+                        }
+                    }
                 }
                 if (
                     role === 'ticket-reviewer' &&

package/engine/models/model-score-sides.ts

@@ -0,0 +1,80 @@
+import { sum, type TicketFacts } from './model-score-facts'
+import { EPSILON, MIN_TICKETS, WORSE_BY } from './model-score-rules'
+
+/**
+ * The model score report's sides (#605): the rates a set of tickets is
+ * judged by, the verdict between two sides, and a side as `luca models`
+ * prints it. Pure.
+ */
+
+export type Verdict = 'not enough data' | 'worse' | 'about the same'
+
+/** The rates a model is judged by, for its tickets or its baseline's. */
+export type ScoreSide = {
+    tickets: number
+    /** Weighted first-review findings per 100 changed lines; `null` with no lines. */
+    findings_per_100_lines: number | null
+    /** `null` with no tickets. */
+    fix_rounds_per_ticket: number | null
+}
+
+/** Findings per 100 lines over `tickets`, those with no lines left out. */
+const findingsRate = (tickets: TicketFacts[]): number | null => {
+    const rated = tickets.filter(({ lines }) => lines !== null)
+    const lines = sum(rated.map(({ lines }) => lines ?? 0))
+    return lines === 0
+        ? null
+        : (sum(rated.map(({ findings }) => findings)) * 100) / lines
+}
+
+export const sideOf = (tickets: TicketFacts[]): ScoreSide => ({
+    tickets: tickets.length,
+    findings_per_100_lines: findingsRate(tickets),
+    fix_rounds_per_ticket:
+        tickets.length === 0
+            ? null
+            : sum(tickets.map(({ fix_rounds }) => fix_rounds)) / tickets.length,
+})
+
+/** Whether `value` is more than `WORSE_BY` above `base`. */
+const above = (value: number | null, base: number | null): boolean =>
+    value !== null && base !== null && value > base * (1 + WORSE_BY) + EPSILON
+
+/** Whether either side has fewer than `MIN_TICKETS` tickets. */
+export const tooFew = (a: ScoreSide, b: ScoreSide): boolean =>
+    a.tickets < MIN_TICKETS || b.tickets < MIN_TICKETS
+
+/** Whether `side`'s findings rate or fix rounds per ticket is above `base`'s. */
+export const sideAbove = (side: ScoreSide, base: ScoreSide): boolean =>
+    above(side.findings_per_100_lines, base.findings_per_100_lines) ||
+    above(side.fix_rounds_per_ticket, base.fix_rounds_per_ticket)
+
+/**
+ * The verdict on a cheaper model against its baseline: "not enough data"
+ * while either side has fewer than `MIN_TICKETS` tickets, "worse" when its
+ * findings rate or fix rounds per ticket is more than `WORSE_BY` above the
+ * baseline's, else "about the same". Pure.
+ */
+export const verdictOf = ({
+    side,
+    baseline,
+}: {
+    side: ScoreSide
+    baseline: ScoreSide
+}): Verdict => {
+    if (tooFew(side, baseline)) return 'not enough data'
+    return sideAbove(side, baseline) ? 'worse' : 'about the same'
+}
+
+export const plural = (count: number, word: string): string =>
+    `${count} ${word}${count === 1 ? '' : 's'}`
+
+export const decimal = (value: number | null): string =>
+    value === null ? '-' : value.toFixed(2)
+
+export const sideText = (side: ScoreSide): string =>
+    [
+        plural(side.tickets, 'ticket'),
+        `${decimal(side.findings_per_100_lines)} findings per 100 lines`,
+        `${decimal(side.fix_rounds_per_ticket)} fix rounds per ticket`,
+    ].join(', ')

package/engine/models/model-score.ts

@@ -5,13 +5,21 @@ import {
     type TicketFacts,
 } from './model-score-facts'
 import {
-    EPSILON,
     FINDING_WEIGHTS,
-    MIN_TICKETS,
     SCORED_ROLES,
-    WORSE_BY,
     type ScoredRole,
 } from './model-score-rules'
+import {
+    decimal,
+    plural,
+    sideAbove,
+    sideOf,
+    sideText,
+    tooFew,
+    verdictOf,
+    type ScoreSide,
+    type Verdict,
+} from './model-score-sides'
 
 import {
     EffortLevelSchema,
@@ -28,6 +36,7 @@ export {
     WORSE_BY,
     type ScoredRole,
 } from './model-score-rules'
+export { verdictOf, type ScoreSide, type Verdict } from './model-score-sides'
 
 /**
  * The model score report (#605): how good each routed role's work was on
@@ -37,17 +46,6 @@ export {
  * reads records and never writes.
  */
 
-export type Verdict = 'not enough data' | 'worse' | 'about the same'
-
-/** The rates a model is judged by, for its tickets or its baseline's. */
-export type ScoreSide = {
-    tickets: number
-    /** Weighted first-review findings per 100 changed lines; `null` with no lines. */
-    findings_per_100_lines: number | null
-    /** `null` with no tickets. */
-    fix_rounds_per_ticket: number | null
-}
-
 /** One routed role on one model at one effort, its tickets reviewed at one effort. */
 export type ModelScore = ScoreSide & {
     role: ScoredRole
@@ -71,6 +69,17 @@ export type ModelScore = ScoreSide & {
     baseline: ScoreSide
     /** `null` for Opus, which is the baseline the others are judged by. */
     verdict: Verdict | null
+    /** Its tickets in a model trial's trial group (#698). */
+    trial_tickets: number
+    /**
+     * The role's control group tickets (#698) where Clef picked this model,
+     * at the same effort and reviewed at the same effort.
+     */
+    control: ScoreSide
+    /**
+     * Its trial tickets against its control tickets; `null` with neither.
+     */
+    trial_verdict: Verdict | null
 }
 
 export type EffortPicksVerdict =
@@ -78,17 +87,33 @@ export type EffortPicksVerdict =
     | 'Clef spots hard tickets'
     | 'no sign yet'
 
+/** Clef's confidence bands its effort picks are broken down by (#697). */
+const CONFIDENCE_BANDS = [
+    { band: 'below 0.3', from: -Infinity, below: 0.3 },
+    { band: '0.3 to 0.5', from: 0.3, below: 0.5 },
+    { band: '0.5 and up', from: 0.5, below: Infinity },
+] as const
+
+/** The two sides of Clef's effort picks within one confidence band. */
+export type EffortPicksBand = {
+    band: (typeof CONFIDENCE_BANDS)[number]['band']
+    wanted_more: ScoreSide
+    rest: ScoreSide
+}
+
 /**
  * Clef's shadow-mode effort picks for one writer role (#663): the tickets
- * where its `agent_effort` answer, at or above the run's confidence bar,
- * was higher than the effort the role's first agent ran at, against the
- * tickets where it was the same or lower.
+ * where its first `agent_effort` answer with a confidence, at any
+ * confidence (#697), was higher than the effort the role's first agent ran
+ * at, against the tickets where it was the same or lower, then the same by
+ * Clef's confidence.
  */
 export type EffortPicks = {
     role: ScoredRole
     wanted_more: ScoreSide
     rest: ScoreSide
     verdict: EffortPicksVerdict
+    by_confidence: EffortPicksBand[]
 }
 
 /** The final review's findings over a set of runs. */
@@ -102,6 +127,11 @@ export type ModelReport = {
     scores: ModelScore[]
     /** One per scored role. */
     effort_picks: EffortPicks[]
+    /**
+     * The roles whose every counted ticket with a model reason ran Opus for
+     * its floor (#697): no cheaper model was ever tried.
+     */
+    floor_only: ScoredRole[]
     /** Fresh learner agents per model and effort; the learner isn't scored. */
     learner: { model: ModelName; effort: EffortLevel; agents: number }[]
     /** Runs with any agent on a cheaper model, and runs without. */
@@ -119,60 +149,13 @@ const median = (values: number[]): number | null => {
         : ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2
 }
 
-/** Findings per 100 lines over `tickets`, those with no lines left out. */
-const findingsRate = (tickets: TicketFacts[]): number | null => {
-    const rated = tickets.filter(({ lines }) => lines !== null)
-    const lines = sum(rated.map(({ lines }) => lines ?? 0))
-    return lines === 0
-        ? null
-        : (sum(rated.map(({ findings }) => findings)) * 100) / lines
-}
-
-const sideOf = (tickets: TicketFacts[]): ScoreSide => ({
-    tickets: tickets.length,
-    findings_per_100_lines: findingsRate(tickets),
-    fix_rounds_per_ticket:
-        tickets.length === 0
-            ? null
-            : sum(tickets.map(({ fix_rounds }) => fix_rounds)) / tickets.length,
-})
-
-/** Whether `value` is more than `WORSE_BY` above `base`. */
-const above = (value: number | null, base: number | null): boolean =>
-    value !== null && base !== null && value > base * (1 + WORSE_BY) + EPSILON
-
-/** Whether either side has fewer than `MIN_TICKETS` tickets. */
-const tooFew = (a: ScoreSide, b: ScoreSide): boolean =>
-    a.tickets < MIN_TICKETS || b.tickets < MIN_TICKETS
-
-/** Whether `side`'s findings rate or fix rounds per ticket is above `base`'s. */
-const sideAbove = (side: ScoreSide, base: ScoreSide): boolean =>
-    above(side.findings_per_100_lines, base.findings_per_100_lines) ||
-    above(side.fix_rounds_per_ticket, base.fix_rounds_per_ticket)
-
-/**
- * The verdict on a cheaper model against its baseline: "not enough data"
- * while either side has fewer than `MIN_TICKETS` tickets, "worse" when its
- * findings rate or fix rounds per ticket is more than `WORSE_BY` above the
- * baseline's, else "about the same". Pure.
- */
-export const verdictOf = ({
-    side,
-    baseline,
-}: {
-    side: ScoreSide
-    baseline: ScoreSide
-}): Verdict => {
-    if (tooFew(side, baseline)) return 'not enough data'
-    return sideAbove(side, baseline) ? 'worse' : 'about the same'
-}
-
 /**
  * Clef's effort picks for `role` over every run's tickets: a ticket wants
  * more when its pick is above the effort the role's first agent ran at, in
  * `EffortLevelSchema`'s order. "Clef spots hard tickets" when the "wanted
  * more" side's findings rate or fix rounds per ticket is more than
- * `WORSE_BY` above the rest's, with `MIN_TICKETS` on each side.
+ * `WORSE_BY` above the rest's, with `MIN_TICKETS` on each side. Each band
+ * holds the tickets whose pick's confidence is at or above its lower edge.
  */
 const effortPicksOf = ({
     tickets,
@@ -188,12 +171,23 @@ const effortPicksOf = ({
         const ran = ticket.efforts[role]
         return pick === undefined || ran === undefined
             ? []
-            : [{ ticket, wants_more: rank(pick) > rank(ran) }]
+            : [
+                  {
+                      ticket,
+                      confidence: pick.confidence,
+                      wants_more: rank(pick.effort) > rank(ran),
+                  },
+              ]
     })
-    const sideWhere = (wants_more: boolean): ScoreSide =>
+    const sideWhere = (
+        wants_more: boolean,
+        within: (confidence: number) => boolean = () => true
+    ): ScoreSide =>
         sideOf(
             picked.flatMap((entry) =>
-                entry.wants_more === wants_more ? [entry.ticket] : []
+                entry.wants_more === wants_more && within(entry.confidence)
+                    ? [entry.ticket]
+                    : []
             )
         )
     const wanted_more = sideWhere(true)
@@ -204,7 +198,35 @@ const effortPicksOf = ({
             ? 'Clef spots hard tickets'
             : 'no sign yet'
     }
-    return { role, wanted_more, rest, verdict: verdictOn() }
+    const by_confidence = CONFIDENCE_BANDS.map(({ band, from, below }) => {
+        const within = (confidence: number) =>
+            confidence >= from && confidence < below
+        return {
+            band,
+            wanted_more: sideWhere(true, within),
+            rest: sideWhere(false, within),
+        }
+    })
+    return { role, wanted_more, rest, verdict: verdictOn(), by_confidence }
+}
+
+/**
+ * Whether every one of `role`'s tickets that has a model reason ran Opus
+ * for its floor, with at least one such ticket. A `null` reason, from
+ * before #603, counts neither way.
+ */
+const isFloorOnly = ({
+    tickets,
+    role,
+}: {
+    tickets: TicketFacts[]
+    role: ScoredRole
+}): boolean => {
+    const reasons = tickets.flatMap(({ models, model_reasons }) => {
+        const reason = model_reasons[role] ?? null
+        return models[role] === undefined || reason === null ? [] : [reason]
+    })
+    return reasons.length > 0 && reasons.every((reason) => reason === 'floor')
 }
 
 /**
@@ -245,6 +267,15 @@ const scoreOf = ({
               )
     const side = sideOf(group)
     const baseline_side = sideOf(baseline)
+    const trial = group.filter(
+        ({ trial_groups }) => trial_groups[role] === 'trial'
+    )
+    const control = sideOf(
+        alike.filter(
+            ({ trial_groups, picks }) =>
+                trial_groups[role] === 'control' && picks[role]?.model === model
+        )
+    )
     return {
         role,
         model,
@@ -269,6 +300,12 @@ const scoreOf = ({
             model === 'opus'
                 ? null
                 : verdictOf({ side, baseline: baseline_side }),
+        trial_tickets: trial.length,
+        control,
+        trial_verdict:
+            trial.length === 0 && control.tickets === 0
+                ? null
+                : verdictOf({ side: sideOf(trial), baseline: control }),
     }
 }
 
@@ -319,7 +356,10 @@ export const scoreModels = ({
                 )
             )
         )
-    ).filter(({ tickets, baseline }) => tickets > 0 || baseline.tickets > 0)
+    ).filter(
+        ({ tickets, baseline, control }) =>
+            tickets > 0 || baseline.tickets > 0 || control.tickets > 0
+    )
     const learner = facts.flatMap((run) => run.learner)
     return {
         runs: runs.length,
@@ -327,6 +367,9 @@ export const scoreModels = ({
         effort_picks: SCORED_ROLES.map((role) =>
             effortPicksOf({ tickets, role })
         ),
+        floor_only: SCORED_ROLES.filter((role) =>
+            isFloorOnly({ tickets, role })
+        ),
         learner: ModelNameSchema.options
             .flatMap((model) =>
                 levels.map((effort) => ({
@@ -348,19 +391,6 @@ export const scoreModels = ({
     }
 }
 
-const plural = (count: number, word: string): string =>
-    `${count} ${word}${count === 1 ? '' : 's'}`
-
-const decimal = (value: number | null): string =>
-    value === null ? '-' : value.toFixed(2)
-
-const sideText = (side: ScoreSide): string =>
-    [
-        plural(side.tickets, 'ticket'),
-        `${decimal(side.findings_per_100_lines)} findings per 100 lines`,
-        `${decimal(side.fix_rounds_per_ticket)} fix rounds per ticket`,
-    ].join(', ')
-
 const scoreText = (score: ModelScore): string[] => {
     const other = SCORED_ROLES.find((role) => role !== score.role)
     const tokens = score.tokens_per_finished_ticket
@@ -368,6 +398,9 @@ const scoreText = (score: ModelScore): string[] => {
         `  ${score.model} at ${score.effort}: ${[
             sideText(score),
             `${score.stuck_tickets} stuck`,
+            ...(score.trial_tickets === 0
+                ? []
+                : [plural(score.trial_tickets, 'trial')]),
             `${tokens === null ? '-' : Math.round(tokens)} tokens per finished ticket`,
             `median confidence ${decimal(score.median_confidence)}`,
             `${other} cheaper on ${score.other_role_cheaper}`,
@@ -379,16 +412,43 @@ const scoreText = (score: ModelScore): string[] => {
                   `    baseline (Clef picked ${score.model} in shadow mode, ran opus at ${score.effort}): ${sideText(score.baseline)}`,
                   `    verdict: ${score.verdict}`,
               ]),
+        ...(score.trial_verdict === null
+            ? []
+            : [
+                  `    control (Clef picked ${score.model}, left out of the trial, ran opus at ${score.effort}): ${sideText(score.control)}`,
+                  `    trial verdict: ${score.trial_verdict}`,
+              ]),
     ]
 }
 
 const effortPicksText = (picks: EffortPicks): string[] => [
-    "  Clef's effort picks (shadow mode)",
+    "  Clef's effort picks (shadow mode), every answer",
     `    wanted more effort than it ran at: ${sideText(picks.wanted_more)}`,
     `    the rest: ${sideText(picks.rest)}`,
     `    verdict: ${picks.verdict}`,
+    "    by Clef's confidence:",
+    ...picks.by_confidence.map(
+        ({ band, wanted_more, rest }) =>
+            `      ${band}: wanted more ${sideText(wanted_more)}; the rest ${sideText(rest)}`
+    ),
 ]
 
+const FLOOR_LINE =
+    "  no cheaper model was ever tried: this role's floor is opus in every run. Set models.floors in .luca/config.json to let it."
+
+/**
+ * A role's model lines: for a floor-only role, its Opus lines and the floor
+ * line in place of the rest.
+ */
+const scoresText = (scores: ModelScore[], floor_only: boolean): string[] => {
+    if (scores.length === 0) return ['  no tickets yet']
+    if (!floor_only) return scores.flatMap(scoreText)
+    return [
+        ...scores.filter(({ model }) => model === 'opus').flatMap(scoreText),
+        FLOOR_LINE,
+    ]
+}
+
 const finalText = (label: string, side: FinalReviewSide): string =>
     `${decimal(side.findings_per_100_lines)} ${label} (${plural(side.runs, 'run')})`
 
@@ -411,9 +471,7 @@ export const formatModelReport = ({
         const picks = report.effort_picks.find((found) => found.role === role)
         return [
             role,
-            ...(scores.length === 0
-                ? ['  no tickets yet']
-                : scores.flatMap(scoreText)),
+            ...scoresText(scores, report.floor_only.includes(role)),
             ...(picks === undefined ? [] : effortPicksText(picks)),
         ]
     })

package/engine/testing/effort-pick-fixtures.ts

@@ -1,4 +1,5 @@
 import {
+    roleSection,
     runOf,
     ticketRun,
     type Built,
@@ -121,19 +122,17 @@ export const runWithBar = (
 export const effortPicksOf = (report: ModelReport, role: Built) =>
     report.effort_picks.find((picks) => picks.role === role)
 
+/** The heading of a role's "Clef's effort picks" section, as printed. */
+export const PICKS_HEADING = "  Clef's effort picks (shadow mode), every answer"
+
 /**
  * The lines of `role`'s "Clef's effort picks (shadow mode)" section in the
  * printed report: the lines after its heading, within the role's part.
  */
 export const pickSection = (text: string, role: Built): string[] => {
-    const lines = text.split('\n')
-    const start = lines.indexOf(role)
-    const end = lines.findIndex(
-        (line, at) => at > start && line !== '' && !line.startsWith(' ')
-    )
-    const part = start < 0 ? [] : lines.slice(start + 1, end)
-    const heading = part.findIndex(
-        (line) => line.trim() === "Clef's effort picks (shadow mode)"
+    const part = roleSection(text, role)
+    const heading = part.findIndex((line) =>
+        line.trim().startsWith("Clef's effort picks (shadow mode)")
     )
     return heading < 0 ? [] : part.slice(heading + 1)
 }

package/engine/testing/model-score-fixtures.ts

@@ -12,6 +12,7 @@ import {
 import { recordsFrom } from './intake-fixtures'
 
 import type { AgentRole } from '../agents/role-results'
+import type { ModelReason } from '../jev/model-pick'
 import type { JournalEntry, JournalRecord } from '../journal/journal-record'
 import type { ModelScore } from '../models/model-score'
 
@@ -29,6 +30,8 @@ export type Built = 'test-writer' | 'implementer'
 /**
  * An agent turn started on `model`; `null` writes one from before #600. It
  * ran at `effort`; left out writes one from before #659, with no effort.
+ * `model_reason` is why it got its model; left out writes one from before
+ * #603, with no reason.
  */
 export const started = ({
     ticket,
@@ -36,12 +39,14 @@ export const started = ({
     model,
     follow_up_of,
     effort,
+    model_reason,
 }: {
     ticket: number | null
     role: AgentRole
     model: Model | null
     follow_up_of?: string
     effort?: Effort
+    model_reason?: ModelReason | null
 }): JournalEntry => ({
     kind: 'agent_started',
     ticket,
@@ -52,6 +57,7 @@ export const started = ({
         follow_up_of: follow_up_of ?? null,
         ...(model === null ? {} : { model }),
         ...(effort === undefined ? {} : { effort }),
+        ...(model_reason === undefined ? {} : { model_reason }),
     },
 })
 
@@ -122,7 +128,8 @@ export type Ask = Omit<Parameters<typeof modelAsk>[0], 'ticket'>
  * (opus unless told), the first review's `findings`, then `rounds` fix
  * rounds (one when the findings block), each fix's review finding a
  * should-fix until the last approves, and a join of `lines` lines. Each
- * role's agents run at its effort, left out as before #659.
+ * role's agents run at its effort, left out as before #659, and with its
+ * model reason, left out as before #603.
  */
 export const ticketRun = ({
     ticket,
@@ -130,6 +137,8 @@ export const ticketRun = ({
     impl = 'opus',
     tw_effort,
     impl_effort,
+    tw_reason,
+    impl_reason,
     review_effort,
     asks = [],
     findings = [],
@@ -143,6 +152,8 @@ export const ticketRun = ({
     impl?: Model | null
     tw_effort?: Effort
     impl_effort?: Effort
+    tw_reason?: ModelReason | null
+    impl_reason?: ModelReason | null
     review_effort?: Effort
     asks?: Ask[]
     findings?: Severity[]
@@ -184,6 +195,7 @@ export const ticketRun = ({
                       role: 'test-writer',
                       model: tw,
                       effort: tw_effort,
+                      model_reason: tw_reason,
                   }),
                   agentFailed({
                       ticket,
@@ -197,6 +209,7 @@ export const ticketRun = ({
             role: 'test-writer',
             model: tw_failed_first ? 'opus' : tw,
             effort: tw_effort,
+            model_reason: tw_failed_first ? 'after_failure' : tw_reason,
         }),
         testsWritten({ ticket }),
         agentSession({
@@ -210,6 +223,7 @@ export const ticketRun = ({
             role: 'implementer',
             model: impl,
             effort: impl_effort,
+            model_reason: impl_reason,
         }),
         implemented({ ticket }),
         ...review(findings, 0),
@@ -220,6 +234,7 @@ export const ticketRun = ({
                 model: impl,
                 follow_up_of: 'impl-1',
                 effort: impl_effort,
+                model_reason: impl_reason,
             }),
             implemented({ ticket }),
             ...review(index + 1 < fixes ? ['should_fix'] : [], index + 1),
@@ -291,6 +306,20 @@ export const groups = (scores: ModelScore[]): string[] =>
         .map(({ role, model }) => `${role}/${model}`)
         .sort()
 
+/**
+ * The lines of `role`'s part of the printed report: the lines after its
+ * heading, up to the next line that isn't indented.
+ */
+export const roleSection = (text: string, role: Built): string[] => {
+    const lines = text.split('\n')
+    const start = lines.indexOf(role)
+    if (start < 0) return []
+    const end = lines.findIndex(
+        (line, at) => at > start && line !== '' && !line.startsWith(' ')
+    )
+    return lines.slice(start + 1, end < 0 ? undefined : end)
+}
+
 /** Clef picked `pick` for the test-writer, sure enough to act on it. */
 export const sure = (pick: Model, confidence = 0.9): Ask => ({
     role: 'test-writer',

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@alecsibilia/luca",
     "description": "Luca: turns a spec on GitHub Issues into a reviewed pull request, run from Paseo with Claude Code",
-    "version": "14.0.0-alpha.27",
+    "version": "14.0.0-alpha.28",
     "license": "Apache-2.0",
     "author": "Alec Sibilia <sibilia.alec@gmail.com>",
     "homepage": "https://github.com/asibilia/luca-framework#readme",