@alecsibilia/luca 14.0.0-alpha.2 → 14.0.0-alpha.21

package/engine/jev/decision-model-credentials.ts

@@ -0,0 +1,212 @@
+import { join } from 'node:path'
+
+/**
+ * The decision model's Cloudflare credentials (#534), from **Luca's own env
+ * file**, `$XDG_CONFIG_HOME/luca/.env` (or `~/.config/luca/.env`). The engine
+ * reads it itself, so it works the same for board-started and
+ * terminal-started runs. It never reads a repo's `.env`. A key set in the
+ * process env wins over the file, key by key, for tests and CI.
+ *
+ * ```
+ * CLOUDFLARE_ACCOUNT_ID=your-account-id
+ * CLOUDFLARE_API_TOKEN=your-workers-ai-token
+ * ```
+ *
+ * `CLOUDFLARE_AUTH_TOKEN` (the name Cloudflare's Clef page uses) works as the
+ * token too; `CLOUDFLARE_API_TOKEN` wins when both are set. Nothing here
+ * ever puts the token in a summary or a log line.
+ */
+
+/** The Cloudflare account the decision model runs on. */
+export const CLOUDFLARE_ACCOUNT_ID = 'CLOUDFLARE_ACCOUNT_ID'
+
+/** The Workers AI token, the name wrangler and Cloudflare's tools use. */
+export const CLOUDFLARE_API_TOKEN = 'CLOUDFLARE_API_TOKEN'
+
+/** Another name for the token, from Cloudflare's Clef page. */
+export const CLOUDFLARE_AUTH_TOKEN = 'CLOUDFLARE_AUTH_TOKEN'
+
+/** The keys the decision model needs, in the order they are named. */
+export const DECISION_MODEL_KEYS = [
+    CLOUDFLARE_ACCOUNT_ID,
+    CLOUDFLARE_API_TOKEN,
+] as const
+
+export type DecisionModelKey = (typeof DECISION_MODEL_KEYS)[number]
+
+/** A process env, or a stand-in for one. */
+export type Env = Record<string, string | undefined>
+
+/**
+ * The credentials, or the keys that are missing. `file` is Luca's env file,
+ * read or not.
+ */
+export type DecisionModelCredentials =
+    | { ok: true; account_id: string; api_token: string; file: string }
+    | { ok: false; missing: DecisionModelKey[]; file: string }
+
+/**
+ * Luca's own env file: `$XDG_CONFIG_HOME/luca/.env` when `XDG_CONFIG_HOME`
+ * is set and not empty, else `<home>/.config/luca/.env`. Pure.
+ *
+ * @example
+ * lucaEnvFile({ env: {}, home: '/Users/me' }) // '/Users/me/.config/luca/.env'
+ */
+export const lucaEnvFile = ({
+    env,
+    home,
+}: {
+    env: Env
+    home: string
+}): string => {
+    const xdg = env.XDG_CONFIG_HOME ?? ''
+    return join(xdg === '' ? join(home, '.config') : xdg, 'luca', '.env')
+}
+
+/** A value without one pair of matching quotes around it. */
+const unquote = (value: string): string => {
+    const first = value.at(0)
+    return value.length >= 2 &&
+        (first === '"' || first === "'") &&
+        value.at(-1) === first
+        ? value.slice(1, -1)
+        : value
+}
+
+/**
+ * The keys and values of an env file: `KEY=VALUE` lines, with an optional
+ * `export ` in front and the value trimmed and taken out of one pair of
+ * single or double quotes. Blank lines, `#` comments, and lines with no `=`
+ * or no key are skipped. No escapes and no `${...}`. Pure.
+ *
+ * @example
+ * parseEnvFile('# keys\nexport A="one two"\nB=2') // { A: 'one two', B: '2' }
+ */
+export const parseEnvFile = (text: string): Record<string, string> => {
+    const values: Record<string, string> = {}
+    for (const raw of text.split(/\r?\n/)) {
+        const line = raw.trim()
+        if (line === '' || line.startsWith('#')) continue
+        const at = line.indexOf('=')
+        if (at === -1) continue
+        const key = line
+            .slice(0, at)
+            .trim()
+            .replace(/^export\s+/, '')
+            .trim()
+        if (key === '') continue
+        values[key] = unquote(line.slice(at + 1).trim())
+    }
+    return values
+}
+
+/** A file's text, or `null` when it is missing or can't be read. */
+const readText = async (path: string): Promise<string | null> => {
+    try {
+        const file = Bun.file(path)
+        return (await file.exists()) ? await file.text() : null
+    } catch {
+        return null
+    }
+}
+
+/** The first of `values` that is set and not empty, or `null`. */
+const firstSet = (values: (string | undefined)[]): string | null =>
+    values.find((value) => value !== undefined && value !== '') ?? null
+
+/**
+ * The decision model's credentials: each key from the process env, else
+ * from Luca's env file (`lucaEnvFile`). An empty value is missing. Never
+ * throws: a missing or unreadable file reads as empty.
+ *
+ * @param env - The process env, such as `process.env`. Tests pass their own.
+ * @param home - The home folder, such as `homedir()`. Tests pass a temp one.
+ * @param read_file - Reads the file; `null` for none. Defaults to Bun's.
+ *
+ * @example
+ * const credentials = await loadDecisionModelCredentials({ env: process.env, home: homedir() })
+ * if (!credentials.ok) console.log(`Add ${credentials.missing.join(', ')} to ${credentials.file}`)
+ */
+export const loadDecisionModelCredentials = async ({
+    env,
+    home,
+    read_file,
+}: {
+    env: Env
+    home: string
+    read_file?: (path: string) => Promise<string | null>
+}): Promise<DecisionModelCredentials> => {
+    const file = lucaEnvFile({ env, home })
+    const text = await (read_file ?? readText)(file).catch(() => null)
+    const values = text === null ? {} : parseEnvFile(text)
+    const account_id = firstSet([
+        env[CLOUDFLARE_ACCOUNT_ID],
+        values[CLOUDFLARE_ACCOUNT_ID],
+    ])
+    const api_token = firstSet([
+        env[CLOUDFLARE_API_TOKEN],
+        env[CLOUDFLARE_AUTH_TOKEN],
+        values[CLOUDFLARE_API_TOKEN],
+        values[CLOUDFLARE_AUTH_TOKEN],
+    ])
+    if (account_id !== null && api_token !== null) {
+        return { ok: true, account_id, api_token, file }
+    }
+    const missing = DECISION_MODEL_KEYS.filter((key) =>
+        key === CLOUDFLARE_ACCOUNT_ID ? account_id === null : api_token === null
+    )
+    return { ok: false, missing, file }
+}
+
+/**
+ * Which keys are set and which are missing, and the file: safe to print,
+ * since it never holds a value. Pure.
+ *
+ * @example
+ * describeDecisionModelCredentials(credentials)
+ * // 'CLOUDFLARE_ACCOUNT_ID set, CLOUDFLARE_API_TOKEN missing (Luca's env file: /Users/me/.config/luca/.env)'
+ */
+export const describeDecisionModelCredentials = (
+    credentials: DecisionModelCredentials
+): string => {
+    const missing: string[] = credentials.ok ? [] : credentials.missing
+    const keys = DECISION_MODEL_KEYS.map(
+        (key) => `${key} ${missing.includes(key) ? 'missing' : 'set'}`
+    )
+    return `${keys.join(', ')} (Luca's env file: ${credentials.file})`
+}
+
+/**
+ * The lines to add to Luca's env file for the missing keys, with
+ * placeholders. Pure.
+ *
+ * @example
+ * missingKeyLines({ missing: ['CLOUDFLARE_API_TOKEN'] }) // ['CLOUDFLARE_API_TOKEN=your-workers-ai-token']
+ */
+export const missingKeyLines = ({
+    missing,
+}: {
+    missing: DecisionModelKey[]
+}): string[] =>
+    missing.map((key) =>
+        key === CLOUDFLARE_ACCOUNT_ID
+            ? `${key}=your-account-id`
+            : `${key}=your-workers-ai-token`
+    )
+
+/**
+ * One plain sentence saying the decision model has no credentials: the
+ * file and the keys to add. Never holds a value. Pure.
+ *
+ * @example
+ * noCredentialsText({ missing: ['CLOUDFLARE_API_TOKEN'], file: '/Users/me/.config/luca/.env' })
+ * // 'No Cloudflare credentials: add CLOUDFLARE_API_TOKEN to /Users/me/.config/luca/.env.'
+ */
+export const noCredentialsText = ({
+    missing,
+    file,
+}: {
+    missing: DecisionModelKey[]
+    file: string
+}): string =>
+    `No Cloudflare credentials: add ${missing.join(' and ')} to ${file}.`

package/engine/jev/jev-jobs.ts

@@ -1,8 +1,17 @@
+import chunk from 'lodash/chunk'
 import fromPairs from 'lodash/fromPairs'
 
-import type { JevFixed, JevJob, JevQuestion, JevRequest } from './jev-schemas'
+import {
+    CLEF_MAX_QUESTIONS,
+    type JevFixed,
+    type JevJob,
+    type JevQuestion,
+    type JevRequest,
+} from './jev-schemas'
+import { offeredModels, type RoutedRole } from './model-pick'
 
 import type { Finding } from '../agents/role-results'
+import { DEFAULT_MODEL, modelsOf } from '../config/engine-config'
 import type { EngineAction } from '../core/decide'
 import type { TicketSnapshot } from '../intake/intake-schemas'
 import type { JournalRecord } from '../journal/journal-record'
@@ -23,17 +32,15 @@ export type JevAsk = {
     role: string | null
     request: JevRequest
     fixed: JevFixed
+    /**
+     * Whether the answer can change what the engine does (#603); left out,
+     * a shadow ask.
+     */
+    acted?: boolean
 }
 
-/** The models Jev may pick for a ticket. */
-export const JEV_MODEL_OPTIONS = [
-    'claude-opus-5-5',
-    'claude-sonnet-5',
-    'claude-haiku-4-5',
-] as const
-
-/** The model the engine really uses for every ticket. */
-export const JEV_FIXED_MODEL = 'claude-opus-5-5'
+/** The model the engine falls back on, and every reviewer's. */
+export const JEV_FIXED_MODEL = DEFAULT_MODEL
 
 /**
  * The stock skills Jev may give an agent. The engine gives agents none, so
@@ -133,42 +140,74 @@ const createTicketWorktreeAsks = ({
         })
         .join('\n')
     const base = ticketState({ state, ticket })
-    return [
-        {
-            job: 'ticket_order',
-            ticket,
-            role: null,
-            request: {
-                state: { ...base, candidates: candidateList },
-                questions: {
-                    next: choice({
-                        instructions:
-                            'Which of these tickets should be built next?',
-                        options: candidates.map(({ number }) =>
-                            ticketKey(number)
-                        ),
-                    }),
-                },
+    // With one ticket left the order is obvious, and Clef refuses a
+    // one-option choice (#546), so there's nothing to ask.
+    const orderAsk: JevAsk[] =
+        candidates.length < 2
+            ? []
+            : [
+                  {
+                      job: 'ticket_order',
+                      ticket,
+                      role: null,
+                      request: {
+                          state: { ...base, candidates: candidateList },
+                          questions: {
+                              next: choice({
+                                  instructions:
+                                      'Which of these tickets should be built next?',
+                                  options: candidates.map(({ number }) =>
+                                      ticketKey(number)
+                                  ),
+                              }),
+                          },
+                      },
+                      fixed: { next: ticketKey(ticket) },
+                  },
+              ]
+    return orderAsk
+}
+
+/**
+ * Which model a fresh agent of a routed role should run on: the ticket's
+ * facts, the role, and which try this is (#602). It offers the models at or
+ * above the role's floor (#603), and is acted on when that floor is below
+ * Opus; a floor of Opus is asked with all three, in shadow mode.
+ */
+const agentModelAsk = ({
+    state,
+    ticket,
+    role,
+    attempt,
+}: {
+    state: RunState
+    ticket: number | null
+    role: RoutedRole
+    attempt: number
+}): JevAsk => {
+    const models =
+        state.config === null ? null : modelsOf({ config: state.config })
+    const floor = models?.floors[role] ?? JEV_FIXED_MODEL
+    return {
+        job: 'agent_model',
+        ticket,
+        role,
+        request: {
+            state: {
+                ...ticketState({ state, ticket }),
+                role,
+                try: String(attempt),
             },
-            fixed: { next: ticketKey(ticket) },
-        },
-        {
-            job: 'ticket_model',
-            ticket,
-            role: null,
-            request: {
-                state: base,
-                questions: {
-                    model: choice({
-                        instructions:
-                            'Which model should build this ticket? Pick the cheapest that will get it right.',
-                        options: JEV_MODEL_OPTIONS,
-                    }),
-                },
+            questions: {
+                model: choice({
+                    instructions: `Which model should the ${role} agent run on? Pick the cheapest that will get it right.`,
+                    options: offeredModels({ floor }),
+                }),
             },
-            fixed: { model: JEV_FIXED_MODEL },
         },
-    ]
+        fixed: { model: JEV_FIXED_MODEL },
+        acted: floor !== JEV_FIXED_MODEL,
+    }
 }
 
 const agentSkillsAsk = ({
@@ -200,9 +239,19 @@ const agentSkillsAsk = ({
     ),
 })
 
+/** The roles whose fresh agents get an `agent_model` ask on a ticket. */
+const ROUTED_TICKET_ROLES: readonly RoutedRole[] = [
+    'test-writer',
+    'implementer',
+]
+
+const isRoutedTicketRole = (role: string): role is RoutedRole =>
+    ROUTED_TICKET_ROLES.some((routed) => routed === role)
+
 /**
- * The asks to make before the engine carries out `action`: ticket order and
- * model before a ticket starts, and skills before each agent.
+ * The asks to make before the engine carries out `action`: ticket order
+ * before a ticket starts, skills before each agent, and the model before
+ * each fresh test-writer, implementer, and learner.
  */
 export const jevAsksBefore = ({
     action,
@@ -214,12 +263,30 @@ export const jevAsksBefore = ({
     switch (action.type) {
         case 'create_ticket_worktree':
             return createTicketWorktreeAsks({ state, ticket: action.ticket })
-        case 'launch_agent':
+        case 'launch_agent': {
+            const { ticket, role } = action
+            const failed = state.tickets[ticket]?.failed_tries[role] ?? 0
+            return [
+                ...(isRoutedTicketRole(role)
+                    ? [
+                          agentModelAsk({
+                              state,
+                              ticket,
+                              role,
+                              attempt: failed + 1,
+                          }),
+                      ]
+                    : []),
+                agentSkillsAsk({ state, ticket, role }),
+            ]
+        }
+        case 'launch_learner':
             return [
-                agentSkillsAsk({
+                agentModelAsk({
                     state,
-                    ticket: action.ticket,
-                    role: action.role,
+                    ticket: null,
+                    role: 'learner',
+                    attempt: state.memory.learner.failed_tries + 1,
                 }),
             ]
         default:
@@ -306,17 +373,33 @@ const findingText = ({ title, detail, file }: Finding): string =>
         .filter((part) => part !== null && part !== '')
         .join('\n')
 
+/**
+ * The severity of each finding a ticket reviewer reported, at most
+ * `CLEF_MAX_QUESTIONS` findings to an ask, since Clef takes no more.
+ */
+const findingSeverityAsks = ({
+    record,
+    state,
+}: {
+    record: JournalRecord
+    state: RunState
+}): JevAsk[] => {
+    if (record.kind !== 'agent_finished') return []
+    if (record.content.role !== 'ticket-reviewer') return []
+    return chunk(record.content.result.findings, CLEF_MAX_QUESTIONS).map(
+        (findings) => findingSeverityAsk({ record, state, findings })
+    )
+}
+
 const findingSeverityAsk = ({
     record,
     state,
+    findings,
 }: {
     record: JournalRecord
     state: RunState
-}): JevAsk | null => {
-    if (record.kind !== 'agent_finished') return null
-    if (record.content.role !== 'ticket-reviewer') return null
-    const { findings } = record.content.result
-    if (findings.length === 0) return null
+    findings: Finding[]
+}): JevAsk => {
     const key = (finding: Finding) => `finding_${questionId(finding.id)}`
     return {
         job: 'finding_severity',
@@ -361,9 +444,10 @@ export const jevAsksAfter = ({
     records: JournalRecord[]
     state: RunState
 }): JevAsk[] =>
-    records.flatMap((record) =>
-        [
-            failureAsk({ record, state }),
-            findingSeverityAsk({ record, state }),
-        ].filter((ask): ask is JevAsk => ask !== null)
-    )
+    records.flatMap((record) => {
+        const failure = failureAsk({ record, state })
+        return [
+            ...(failure === null ? [] : [failure]),
+            ...findingSeverityAsks({ record, state }),
+        ]
+    })

package/engine/jev/jev-schemas.ts

@@ -1,9 +1,12 @@
 import { z } from 'zod'
 
 /**
- * Jev, TypeSafe's labeling model, runs in **shadow mode**: the engine asks it
- * and journals its answers, but acts on its own fixed choices. These are the
- * shapes of what the engine asks and what it keeps of each answer.
+ * The **decision model** (Cloudflare's Clef today; Jev, TypeSafe's labeling
+ * model, before #534) runs in **shadow mode**: the engine asks it and
+ * journals its answers, but acts on its own fixed choices. These are the
+ * shapes of what the engine asks and what it keeps of each answer. The
+ * names (and the journal's `jev_*` record kinds) keep the Jev name, so old
+ * journals still read.
  */
 
 /** Pick one of the criteria's keys. */
@@ -44,6 +47,43 @@ export const JevRequestSchema = z.object({
 
 export type JevRequest = z.infer<typeof JevRequestSchema>
 
+/** The most questions Clef takes in one request. */
+export const CLEF_MAX_QUESTIONS = 64
+
+/**
+ * A request Clef will take (#546): 1 to 64 questions, 2 to 255 options per
+ * choice, and 2 to 10 levels per score. Clef refuses anything else, so the
+ * engine checks each ask against this before sending it. Journals keep the
+ * looser `JevRequestSchema`, so old asks still read.
+ */
+export const ClefRequestSchema = z.object({
+    state: z.record(z.string(), z.string()),
+    questions: z
+        .record(
+            z.string(),
+            z.discriminatedUnion('type', [
+                JevChoiceQuestionSchema.refine(
+                    ({ criteria }) => {
+                        const options = Object.keys(criteria).length
+                        return options >= 2 && options <= 255
+                    },
+                    { message: 'A choice takes 2 to 255 options.' }
+                ),
+                JevScoreQuestionSchema,
+                JevNoulQuestionSchema,
+            ])
+        )
+        .refine(
+            (questions) => {
+                const count = Object.keys(questions).length
+                return count >= 1 && count <= CLEF_MAX_QUESTIONS
+            },
+            {
+                message: `A request takes 1 to ${CLEF_MAX_QUESTIONS} questions.`,
+            }
+        ),
+})
+
 /**
  * What the engine keeps of one answer. `value` is the choice, score, or
  * probability Jev gave; `raw` is the answer exactly as it came back.
@@ -57,9 +97,14 @@ export const JevAnswerSchema = z.object({
 
 export type JevAnswer = z.infer<typeof JevAnswerSchema>
 
-/** The jobs the engine asks Jev about. */
+/**
+ * The jobs the engine asks Jev about. `ticket_model` (one model per ticket)
+ * is no longer asked; `agent_model` (one per agent) replaced it in #602. It
+ * stays so old journals read.
+ */
 export const JevJobSchema = z.enum([
     'ticket_model',
+    'agent_model',
     'agent_skills',
     'ticket_order',
     'failure_kind',
@@ -68,9 +113,17 @@ export const JevJobSchema = z.enum([
 
 export type JevJob = z.infer<typeof JevJobSchema>
 
-/** Why a Jev call gave no answers. */
+/**
+ * Why a decision model call gave no answers. New calls write
+ * `missing_credentials` (no Cloudflare account id or token), `rejected`
+ * (Cloudflare turned the token down: HTTP 401 or 403), `timeout`, or
+ * `error`. `missing_key` is Jev's old reason (it had no key), kept so old
+ * journals read.
+ */
 export const JevFailureReasonSchema = z.enum([
     'missing_key',
+    'missing_credentials',
+    'rejected',
     'timeout',
     'error',
 ])
@@ -87,3 +140,31 @@ export const JevFixedSchema = z.record(
 )
 
 export type JevFixed = z.infer<typeof JevFixedSchema>
+
+/** The Clef models on Workers AI the engine can ask (#534). */
+export const DECISION_MODEL_IDS = [
+    '@cf/cloudflare/clef',
+    '@cf/cloudflare/clef-flash',
+] as const
+
+/** A decision model's Workers AI id. */
+export const DecisionModelIdSchema = z.enum(DECISION_MODEL_IDS)
+
+export type DecisionModelId = z.infer<typeof DecisionModelIdSchema>
+
+/** The model the engine asks when the config doesn't say: Clef, the accurate one. */
+export const DEFAULT_DECISION_MODEL: DecisionModelId = '@cf/cloudflare/clef'
+
+/**
+ * Why the decision model is off for a run: `no_credentials` (Luca's env
+ * file and the process env have no Cloudflare account id or token), or
+ * `rejected` (Cloudflare turned the token down on an ask).
+ */
+export const DecisionModelOffReasonSchema = z.enum([
+    'no_credentials',
+    'rejected',
+])
+
+export type DecisionModelOffReason = z.infer<
+    typeof DecisionModelOffReasonSchema
+>