@alecsibilia/luca 14.0.0-alpha.21 → 14.0.0-alpha.22

package/engine/core/execute-memory.ts

@@ -1,6 +1,7 @@
 import type { MemoryAction, MemorySaveStep } from './decide-memory'
 import { buildContext, need, runTurn, type BuildDeps } from './execute-build'
 
+import { roleEffort } from '../agents/role-effort'
 import { launchPick } from '../jev/model-pick'
 import type { Journal } from '../journal/journal'
 import type { JournalRecord } from '../journal/journal-record'
@@ -416,6 +417,10 @@ export const executeMemoryAction = async ({
                 role: 'learner',
                 step,
             })
+            const effort = roleEffort({
+                role: 'learner',
+                config: built.config,
+            })
             journal.append({
                 kind: 'agent_started',
                 ticket: null,
@@ -426,6 +431,7 @@ export const executeMemoryAction = async ({
                     follow_up_of: null,
                     model,
                     model_reason,
+                    effort,
                 },
             })
             await runTurn({
@@ -445,6 +451,7 @@ export const executeMemoryAction = async ({
                         // The learner writes nothing and talks to nobody.
                         messaging: null,
                         model,
+                        effort,
                     }),
             })
             return

package/engine/guards/helper-rules.ts

@@ -6,6 +6,8 @@ import { z } from 'zod'
 
 import { ALLOW, deny, READ_TOOLS, type ToolDecision } from './guard-roles'
 
+import type { EffortLevel } from '../config/engine-config'
+
 /** The SDK tool that starts a helper. */
 export const HELPER_TOOL = 'Agent'
 
@@ -15,6 +17,9 @@ export const HELPER_TYPE = 'searcher'
 /** Helpers always run on the newest Haiku, by its short name. */
 export const HELPER_MODEL = 'haiku'
 
+/** Helpers always run at low effort (#659): a search needs little thought. */
+export const HELPER_EFFORT: EffortLevel = 'low'
+
 /** The only tools a helper has: it reads and searches, nothing else. */
 export const HELPER_TOOLS = READ_TOOLS
 
@@ -26,6 +31,7 @@ export const SEARCHER = {
     description:
         'A read-only helper that searches the repo: finds files, symbols and callers, and reports what it found. It cannot edit files or run commands.',
     model: HELPER_MODEL,
+    effort: HELPER_EFFORT,
     tools: [...HELPER_TOOLS],
     prompt: 'You are a read-only searcher. You may only read and search files in the current worktree with Read, Grep and Glob. Never try to change a file or run a command. Answer with what you found: file paths, line numbers and short excerpts, as briefly as the question allows.',
 }

package/engine/index.ts

@@ -15,7 +15,6 @@ export {
     type AgentQuerySession,
 } from './agents/claude-launcher'
 export {
-    AGENT_EFFORT,
     agentEnv,
     agentOptions,
     BANNED_ENV,
@@ -32,6 +31,7 @@ export {
     LUCA_SERVER,
     sendMessageDescription,
 } from './agents/message-tool'
+export { REVIEW_EFFORT, ROLE_EFFORT, roleEffort } from './agents/role-effort'
 export { roleInstructions } from './agents/role-instructions'
 export {
     rolePrompt,

package/engine/jev/jev-jobs.ts

@@ -10,8 +10,13 @@ import {
 } from './jev-schemas'
 import { offeredModels, type RoutedRole } from './model-pick'
 
+import { ROLE_EFFORT, roleEffort } from '../agents/role-effort'
 import type { Finding } from '../agents/role-results'
-import { DEFAULT_MODEL, modelsOf } from '../config/engine-config'
+import {
+    DEFAULT_MODEL,
+    modelsOf,
+    type EffortLevel,
+} from '../config/engine-config'
 import type { EngineAction } from '../core/decide'
 import type { TicketSnapshot } from '../intake/intake-schemas'
 import type { JournalRecord } from '../journal/journal-record'
@@ -210,6 +215,51 @@ const agentModelAsk = ({
     }
 }
 
+/** The efforts an `agent_effort` ask offers, lowest first. */
+const OFFERED_EFFORTS: readonly EffortLevel[] = ['medium', 'high', 'xhigh']
+
+/**
+ * Which effort a fresh test-writer or implementer on a ticket should use
+ * (#661), with the same facts as its `agent_model` ask. Shadow mode only:
+ * the agent always runs at its role's effort.
+ */
+const agentEffortAsk = ({
+    state,
+    ticket,
+    role,
+    attempt,
+}: {
+    state: RunState
+    ticket: number
+    role: RoutedRole
+    attempt: number
+}): JevAsk => ({
+    job: 'agent_effort',
+    ticket,
+    role,
+    request: {
+        state: {
+            ...ticketState({ state, ticket }),
+            role,
+            try: String(attempt),
+        },
+        questions: {
+            effort: choice({
+                instructions:
+                    'Which effort should this agent use? Pick the lowest that will get it right.',
+                options: OFFERED_EFFORTS,
+            }),
+        },
+    },
+    fixed: {
+        effort:
+            state.config === null
+                ? ROLE_EFFORT[role]
+                : roleEffort({ role, config: state.config }),
+    },
+    acted: false,
+})
+
 const agentSkillsAsk = ({
     state,
     ticket,
@@ -250,8 +300,9 @@ const isRoutedTicketRole = (role: string): role is RoutedRole =>
 
 /**
  * The asks to make before the engine carries out `action`: ticket order
- * before a ticket starts, skills before each agent, and the model before
- * each fresh test-writer, implementer, and learner.
+ * before a ticket starts, skills before each agent, the model before each
+ * fresh test-writer, implementer, and learner, and the effort before each
+ * fresh test-writer and implementer.
  */
 export const jevAsksBefore = ({
     action,
@@ -265,16 +316,12 @@ export const jevAsksBefore = ({
             return createTicketWorktreeAsks({ state, ticket: action.ticket })
         case 'launch_agent': {
             const { ticket, role } = action
-            const failed = state.tickets[ticket]?.failed_tries[role] ?? 0
+            const attempt = (state.tickets[ticket]?.failed_tries[role] ?? 0) + 1
             return [
                 ...(isRoutedTicketRole(role)
                     ? [
-                          agentModelAsk({
-                              state,
-                              ticket,
-                              role,
-                              attempt: failed + 1,
-                          }),
+                          agentModelAsk({ state, ticket, role, attempt }),
+                          agentEffortAsk({ state, ticket, role, attempt }),
                       ]
                     : []),
                 agentSkillsAsk({ state, ticket, role }),

package/engine/jev/jev-schemas.ts

@@ -100,11 +100,13 @@ export type JevAnswer = z.infer<typeof JevAnswerSchema>
 /**
  * The jobs the engine asks Jev about. `ticket_model` (one model per ticket)
  * is no longer asked; `agent_model` (one per agent) replaced it in #602. It
- * stays so old journals read.
+ * stays so old journals read. `agent_effort` is only ever asked in shadow
+ * mode (#661).
  */
 export const JevJobSchema = z.enum([
     'ticket_model',
     'agent_model',
+    'agent_effort',
     'agent_skills',
     'ticket_order',
     'failure_kind',

package/engine/journal/agent-entries.ts

@@ -8,6 +8,7 @@ import {
     FileModeSchema,
     RoleResultSchema,
 } from '../agents/role-results'
+import { EffortLevelSchema } from '../config/engine-config'
 import { GateCheckSchema } from '../gates/gate-schemas'
 import { ModelReasonSchema } from '../jev/model-pick'
 
@@ -23,7 +24,9 @@ import { ModelReasonSchema } from '../jev/model-pick'
  * follow-up's is its session's); `null` in journals from before #600. The
  * exact version is in the turn's `agent_session`. `model_reason` is why it
  * launched on that model (#603; a follow-up's is its session's); `null` in
- * journals from before it.
+ * journals from before it. `effort` is the effort it ran at (#659; a
+ * follow-up's is its session's); `null` in journals from before it, when
+ * every agent ran at `high`.
  */
 export const AgentStartedEntrySchema = z.object({
     ...ENTRY_FIELDS,
@@ -34,6 +37,7 @@ export const AgentStartedEntrySchema = z.object({
         follow_up_of: z.string().nullable().default(null),
         model: z.string().nullable().default(null),
         model_reason: ModelReasonSchema.nullable().default(null),
+        effort: EffortLevelSchema.nullable().default(null),
     }),
 })
 

package/engine/models/model-score-facts.ts

@@ -0,0 +1,273 @@
+import {
+    EFFORT_BEFORE_659,
+    FINDING_WEIGHTS,
+    SCORED_ROLES,
+    type ScoredRole,
+} from './model-score-rules'
+
+import type { FindingSeverity } from '../agents/role-results'
+import { modelsOf } from '../config/config-fields'
+import {
+    EffortLevelSchema,
+    ModelNameSchema,
+    ModelsConfigSchema,
+    type EffortLevel,
+    type ModelName,
+} from '../config/engine-config'
+import type { JournalRecord } from '../journal/journal-record'
+import { countedTokens } from '../limits/plan-usage'
+
+/**
+ * What one run's journal says for the model score report (#605): each
+ * ticket's models, efforts, Clef's picks and review results, the learner's
+ * agents, and the final review's findings. Pure: it only reads records.
+ */
+
+/** Clef's first model pick for a role on a ticket. */
+export type Pick = {
+    model: ModelName | null
+    confidence: number | null
+    /** Asked in shadow mode: `acted: false`, or an older `ticket_model` ask. */
+    shadow: boolean
+}
+
+/** What one run's journal says about one ticket. */
+export type TicketFacts = {
+    /** The model of each role's first agent. */
+    models: Partial<Record<ScoredRole, ModelName>>
+    /** The effort of each role's first agent, an older one's `high`. */
+    efforts: Partial<Record<ScoredRole, EffortLevel>>
+    /** The first ticket review's effort, `high` when none is journaled. */
+    review_effort: EffortLevel
+    picks: Partial<Record<ScoredRole, Pick>>
+    /**
+     * Clef's first `agent_effort` answer for each role at or above the run's
+     * confidence bar.
+     */
+    effort_picks: Partial<Record<ScoredRole, EffortLevel>>
+    reviews: number
+    /** The first ticket review's weighted findings. */
+    findings: number
+    fix_rounds: number
+    joined: boolean
+    lines: number | null
+    stuck: boolean
+    tokens: number | null
+}
+
+/** A fresh learner agent's model and effort. */
+export type LearnerRun = { model: ModelName; effort: EffortLevel }
+
+/** What one run's journal says. */
+export type RunFacts = {
+    tickets: TicketFacts[]
+    routed: boolean
+    learner: LearnerRun[]
+    /** The first final review round's weighted findings, or `null` with none. */
+    final_findings: number | null
+    /** The joined tickets' lines, or `null` when one has none. */
+    lines: number | null
+}
+
+/**
+ * The model family a short name or full model id (as older `ticket_model`
+ * asks named them) belongs to, or `null` for none.
+ *
+ * @example
+ * familyOf('sonnet') // 'sonnet'
+ */
+const familyOf = (name: unknown): ModelName | null => {
+    if (typeof name !== 'string') return null
+    const lower = name.toLowerCase()
+    return (
+        ModelNameSchema.options.find((model) => lower.includes(model)) ?? null
+    )
+}
+
+const isScored = (role: string | null): role is ScoredRole =>
+    SCORED_ROLES.some((scored) => scored === role)
+
+export const sum = (values: number[]): number =>
+    values.reduce((total, value) => total + value, 0)
+
+const weigh = (counts: Record<FindingSeverity, number>): number =>
+    counts.blocker * FINDING_WEIGHTS.blocker +
+    counts.should_fix * FINDING_WEIGHTS.should_fix +
+    counts.nit * FINDING_WEIGHTS.nit
+
+const newTicket = (): TicketFacts => ({
+    models: {},
+    efforts: {},
+    review_effort: EFFORT_BEFORE_659,
+    picks: {},
+    effort_picks: {},
+    reviews: 0,
+    findings: 0,
+    fix_rounds: 0,
+    joined: false,
+    lines: null,
+    stuck: false,
+    tokens: null,
+})
+
+/**
+ * What one run's records say, ticket by ticket. An agent with no model ran
+ * Opus (before #600), and one with no effort ran at `high` (before #659).
+ */
+export const factsOf = (records: JournalRecord[]): RunFacts => {
+    const tickets = new Map<number, TicketFacts>()
+    const ticketOf = (number: number): TicketFacts => {
+        const found = tickets.get(number) ?? newTicket()
+        tickets.set(number, found)
+        return found
+    }
+    const asks = new Map<number, { role: string | null; shadow: boolean }>()
+    const effort_asks = new Map<number, string | null>()
+    const reviewed = new Set<number>()
+    const learner: LearnerRun[] = []
+    let bar = ModelsConfigSchema.parse({}).confidence_bar
+    let routed = false
+    let final_rounds = 0
+    let final_findings = 0
+    for (const record of records) {
+        switch (record.kind) {
+            case 'run_started':
+                bar = modelsOf({ config: record.content.config }).confidence_bar
+                break
+            case 'jev_asked':
+                if (record.content.job === 'agent_effort') {
+                    effort_asks.set(record.seq, record.role)
+                }
+                if (
+                    record.content.job === 'agent_model' ||
+                    record.content.job === 'ticket_model'
+                ) {
+                    asks.set(record.seq, {
+                        role: record.role,
+                        shadow:
+                            record.content.job === 'ticket_model' ||
+                            !record.content.acted,
+                    })
+                }
+                break
+            case 'jev_answered': {
+                const effort_role = effort_asks.get(record.content.asked_seq)
+                if (effort_role !== undefined && record.ticket !== null) {
+                    const answer = record.content.answers.effort
+                    const level = EffortLevelSchema.safeParse(answer?.value)
+                    const confidence = answer?.confidence ?? null
+                    if (
+                        isScored(effort_role) &&
+                        level.success &&
+                        confidence !== null &&
+                        confidence >= bar
+                    ) {
+                        ticketOf(record.ticket).effort_picks[effort_role] ??=
+                            level.data
+                    }
+                    break
+                }
+                const ask = asks.get(record.content.asked_seq)
+                if (ask === undefined || record.ticket === null) break
+                const answer = record.content.answers.model
+                const pick: Pick = {
+                    model: familyOf(answer?.value),
+                    confidence: answer?.confidence ?? null,
+                    shadow: ask.shadow,
+                }
+                const roles =
+                    record.content.job === 'ticket_model'
+                        ? SCORED_ROLES
+                        : SCORED_ROLES.filter((role) => role === ask.role)
+                const facts = ticketOf(record.ticket)
+                for (const role of roles) facts.picks[role] ??= pick
+                break
+            }
+            case 'agent_started': {
+                const { role, model, effort, follow_up_of } = record.content
+                const family = familyOf(model) ?? 'opus'
+                const level = effort ?? EFFORT_BEFORE_659
+                if (follow_up_of !== null) break
+                if (role === 'learner') {
+                    learner.push({ model: family, effort: level })
+                }
+                if (isScored(role) || role === 'learner') {
+                    routed ||= family !== 'opus'
+                }
+                if (record.ticket === null) break
+                if (isScored(role)) {
+                    const facts = ticketOf(record.ticket)
+                    facts.models[role] ??= family
+                    facts.efforts[role] ??= level
+                }
+                if (
+                    role === 'ticket-reviewer' &&
+                    !reviewed.has(record.ticket)
+                ) {
+                    reviewed.add(record.ticket)
+                    ticketOf(record.ticket).review_effort = level
+                }
+                break
+            }
+            case 'agent_finished': {
+                const { content } = record
+                if (
+                    content.role !== 'ticket-reviewer' ||
+                    record.ticket === null
+                ) {
+                    break
+                }
+                const facts = ticketOf(record.ticket)
+                if (facts.reviews === 0) {
+                    facts.findings = sum(
+                        content.result.findings.map(
+                            ({ severity }) => FINDING_WEIGHTS[severity]
+                        )
+                    )
+                }
+                facts.reviews += 1
+                if (content.result.verdict === 'changes_requested') {
+                    facts.fix_rounds += 1
+                }
+                break
+            }
+            case 'ticket_joined':
+                if (record.ticket !== null && record.content.ok) {
+                    const facts = ticketOf(record.ticket)
+                    facts.joined = true
+                    facts.lines = record.content.lines_changed
+                }
+                break
+            case 'ticket_stuck':
+                if (record.ticket !== null) ticketOf(record.ticket).stuck = true
+                break
+            case 'usage_recorded': {
+                const { scope, ticket, tokens } = record.content
+                if (scope === 'ticket' && ticket !== null) {
+                    ticketOf(ticket).tokens = countedTokens({ tokens })
+                }
+                break
+            }
+            case 'final_review_started':
+                final_rounds += 1
+                break
+            case 'lens_finished':
+                if (final_rounds === 1) {
+                    final_findings += weigh(record.content.findings)
+                }
+                break
+            default:
+                break
+        }
+    }
+    const joined = [...tickets.values()].filter(({ joined }) => joined)
+    return {
+        tickets: [...tickets.values()],
+        routed,
+        learner,
+        final_findings: final_rounds > 0 ? final_findings : null,
+        lines: joined.some(({ lines }) => lines === null)
+            ? null
+            : sum(joined.map(({ lines }) => lines ?? 0)),
+    }
+}

package/engine/models/model-score-rules.ts

@@ -0,0 +1,35 @@
+import type { FindingSeverity } from '../agents/role-results'
+import type { EffortLevel } from '../config/engine-config'
+
+/**
+ * The model score report's constants (#605, #662): the finding weights, the
+ * verdict edges, the scored roles, and the effort an older journal's agent
+ * ran at. The one place they live.
+ */
+
+/** How much each ticket review finding weighs. */
+export const FINDING_WEIGHTS: Record<FindingSeverity, number> = {
+    blocker: 3,
+    should_fix: 1,
+    nit: 0,
+}
+
+/** The fewest tickets on each side before a verdict is given. */
+export const MIN_TICKETS = 10
+
+/** How far above the baseline a rate may be before it is "worse". */
+export const WORSE_BY = 0.2
+
+/** Rounding slack when a rate is compared with the baseline's. */
+export const EPSILON = 1e-9
+
+/** The roles scored by the ticket review's findings. */
+export const SCORED_ROLES = ['test-writer', 'implementer'] as const
+
+export type ScoredRole = (typeof SCORED_ROLES)[number]
+
+/**
+ * The effort an agent with no journaled effort ran at: every role ran at
+ * `high` before #659.
+ */
+export const EFFORT_BEFORE_659: EffortLevel = 'high'