@alecsibilia/luca 14.0.0-alpha.2 → 14.0.0-alpha.21

package/engine/jev/model-pick.ts

@@ -0,0 +1,275 @@
+import { z } from 'zod'
+
+import type { AgentRole } from '../agents/role-results'
+import {
+    DEFAULT_MODEL,
+    ModelNameSchema,
+    modelsOf,
+    ROUTED_ROLES,
+    type EngineConfig,
+    type ModelName,
+    type ModelsConfig,
+} from '../config/engine-config'
+import type { JournalRecord } from '../journal/journal-record'
+import type { RunState } from '../journal/replay'
+
+/**
+ * Which model each fresh agent launches on (#603): the pick rule, and the
+ * decision model's answer it reads from the journal.
+ */
+
+/**
+ * Why an agent launched on its model:
+ * - `locked`: a reviewer, a lens, or a final-review fixer, always Opus.
+ * - `after_failure`: its role already failed a try on its ticket on a
+ *   cheaper model (#604).
+ * - `floor`: its role's floor is Opus.
+ * - `no_answer`: the decision model gave no answer (off, timed out,
+ *   errored, or the ask was skipped).
+ * - `unsure`: its confidence was below the bar or missing, or it picked a
+ *   model the ask didn't offer.
+ * - `decision_model`: the decision model's pick.
+ */
+export const ModelReasonSchema = z.enum([
+    'locked',
+    'after_failure',
+    'floor',
+    'no_answer',
+    'unsure',
+    'decision_model',
+])
+
+export type ModelReason = z.infer<typeof ModelReasonSchema>
+
+/** The model an agent launches on, and why. */
+export type ModelPick = { model: ModelName; model_reason: ModelReason }
+
+/** Reviewers, lenses, and the final review's fixers: always Opus. */
+export const LOCKED_PICK: ModelPick = {
+    model: DEFAULT_MODEL,
+    model_reason: 'locked',
+}
+
+export type RoutedRole = (typeof ROUTED_ROLES)[number]
+
+/** Whether a role has a model floor, so the decision model may pick its model. */
+export const isRoutedRole = (role: string): role is RoutedRole =>
+    ROUTED_ROLES.some((routed) => routed === role)
+
+/**
+ * The models an `agent_model` ask offers for a role with this floor: the
+ * floor and every stronger one. A floor of `opus` offers all three, since
+ * that ask is shadow only.
+ *
+ * @example
+ * offeredModels({ floor: 'sonnet' }) // ['opus', 'sonnet']
+ */
+export const offeredModels = ({ floor }: { floor: ModelName }): ModelName[] => {
+    const all = ModelNameSchema.options
+    return floor === 'opus' ? [...all] : all.slice(0, all.indexOf(floor) + 1)
+}
+
+/** The decision model's answer to an `agent_model` ask, and what it offered. */
+export type ModelAnswer = {
+    offered: string[]
+    pick: string | number | null
+    confidence: number | null
+}
+
+/**
+ * The model a fresh agent launches on. Pure; the first match wins: a
+ * reviewer or lens is `locked`; a role that failed a try on its ticket on a
+ * cheaper model is `after_failure`; a floor of `opus` is `floor`; no answer
+ * is `no_answer`; an answer below the bar, with no confidence, or not
+ * offered is `unsure`; else the decision model's pick. Every fallback is
+ * Opus.
+ *
+ * @example
+ * pickModel({ role: 'test-writer', models, failed_cheaper: false, answer: { offered: ['sonnet', 'opus'], pick: 'sonnet', confidence: 0.9 } })
+ * // { model: 'sonnet', model_reason: 'decision_model' } with a sonnet floor and the 0.8 bar
+ */
+export const pickModel = ({
+    role,
+    models,
+    failed_cheaper,
+    answer,
+}: {
+    role: AgentRole
+    models: ModelsConfig
+    /** Whether the role already failed a try on its ticket on a cheaper model. */
+    failed_cheaper: boolean
+    answer: ModelAnswer | null
+}): ModelPick => {
+    if (!isRoutedRole(role)) return LOCKED_PICK
+    if (failed_cheaper) return { model: 'opus', model_reason: 'after_failure' }
+    if (models.floors[role] === 'opus') {
+        return { model: 'opus', model_reason: 'floor' }
+    }
+    if (answer === null) return { model: 'opus', model_reason: 'no_answer' }
+    const pick = ModelNameSchema.safeParse(answer.pick)
+    if (
+        !pick.success ||
+        !answer.offered.includes(pick.data) ||
+        answer.confidence === null ||
+        answer.confidence < models.confidence_bar
+    ) {
+        return { model: 'opus', model_reason: 'unsure' }
+    }
+    return { model: pick.data, model_reason: 'decision_model' }
+}
+
+/**
+ * The answer to the `agent_model` ask journaled for this agent since its
+ * role's last start on the ticket, or `null` with none: no ask (the decision
+ * model is off, or the ask was skipped), or an ask that failed.
+ */
+export const modelAnswerOf = ({
+    records,
+    ticket,
+    role,
+}: {
+    records: JournalRecord[]
+    ticket: number | null
+    role: AgentRole
+}): ModelAnswer | null => {
+    const lastStart = records.findLastIndex(
+        (record) =>
+            record.kind === 'agent_started' &&
+            record.ticket === ticket &&
+            record.role === role
+    )
+    const asked = records
+        .slice(lastStart + 1)
+        .findLast(
+            (record) =>
+                record.kind === 'jev_asked' &&
+                record.content.job === 'agent_model' &&
+                record.ticket === ticket &&
+                record.role === role
+        )
+    if (asked?.kind !== 'jev_asked') return null
+    const answered = records.find(
+        (record) =>
+            record.kind === 'jev_answered' &&
+            record.content.asked_seq === asked.seq
+    )
+    if (answered?.kind !== 'jev_answered') return null
+    const answer = answered.content.answers.model
+    if (answer === undefined) return null
+    const question = asked.content.request.questions.model
+    return {
+        offered:
+            question?.type === 'choice' ? Object.keys(question.criteria) : [],
+        pick: answer.value,
+        confidence: answer.confidence,
+    }
+}
+
+/**
+ * Whether this role failed a try on this ticket on a model cheaper than
+ * Opus (#604), as replay counts a try (`failed_cheaper`): the learner's at
+ * the end of a run, with no ticket.
+ */
+export const failedCheaper = ({
+    state,
+    ticket,
+    role,
+}: {
+    state: RunState
+    ticket: number | null
+    role: string
+}): boolean =>
+    ticket === null
+        ? role === 'learner' && state.memory.learner.failed_cheaper
+        : (state.tickets[ticket]?.failed_cheaper ?? []).some(
+              (failed) => failed === role
+          )
+
+/**
+ * The model a launch step's earlier try journaled before a crash cut it
+ * off, so its redo launches on it again (#604); `null` if none did.
+ *
+ * @param first_seq - The seq of the step's first try.
+ */
+export const redonePick = ({
+    records,
+    first_seq,
+    ticket,
+    role,
+}: {
+    records: JournalRecord[]
+    first_seq: number
+    ticket: number | null
+    role: string
+}): ModelPick | null => {
+    const started = records.findLast(
+        (record) =>
+            record.seq >= first_seq &&
+            record.kind === 'agent_started' &&
+            record.ticket === ticket &&
+            record.content.role === role
+    )
+    if (started?.kind !== 'agent_started') return null
+    const model = ModelNameSchema.safeParse(started.content.model)
+    const { model_reason } = started.content
+    if (!model.success || model_reason === null) return null
+    return { model: model.data, model_reason }
+}
+
+/** Which try of its step a launch is: a redo names its first try's seq. */
+export type LaunchTry = { first_seq: number; redo: boolean }
+
+/**
+ * Whether a fresh agent's launch asks the decision model for its model: not
+ * after its role failed a try on a cheaper model, and not on a redo whose
+ * first try journaled its model.
+ */
+export const asksModel = ({
+    records,
+    state,
+    ticket,
+    role,
+    step,
+}: {
+    records: JournalRecord[]
+    state: RunState
+    ticket: number | null
+    role: string
+    step: LaunchTry | undefined
+}): boolean =>
+    !(
+        (step?.redo === true &&
+            redonePick({ records, first_seq: step.first_seq, ticket, role }) !==
+                null) ||
+        failedCheaper({ state, ticket, role })
+    )
+
+/**
+ * The model a fresh agent launches on, from the run's config and the
+ * answer the launch step just journaled. A redo launches on the model its
+ * first try journaled.
+ */
+export const launchPick = ({
+    records,
+    state,
+    config,
+    ticket,
+    role,
+    step,
+}: {
+    records: JournalRecord[]
+    state: RunState
+    config: EngineConfig
+    ticket: number | null
+    role: AgentRole
+    step: LaunchTry
+}): ModelPick =>
+    (step.redo
+        ? redonePick({ records, first_seq: step.first_seq, ticket, role })
+        : null) ??
+    pickModel({
+        role,
+        models: modelsOf({ config }),
+        failed_cheaper: failedCheaper({ state, ticket, role }),
+        answer: modelAnswerOf({ records, ticket, role }),
+    })

package/engine/journal/agent-entries.ts

@@ -0,0 +1,167 @@
+import { z } from 'zod'
+
+import { ENTRY_FIELDS } from './entry-fields'
+
+import { AgentSessionSchema } from '../agents/agent-launcher'
+import {
+    AgentRoleSchema,
+    FileModeSchema,
+    RoleResultSchema,
+} from '../agents/role-results'
+import { GateCheckSchema } from '../gates/gate-schemas'
+import { ModelReasonSchema } from '../jev/model-pick'
+
+/**
+ * The journal entries of agent turns: started, finished or failed, their
+ * sessions, and shared `.git` changes during a turn.
+ */
+
+/**
+ * An agent turn was started, with its prompt word for word. A follow-up in a
+ * fix loop names the session it went to in `follow_up_of`, and `prompt` holds
+ * the follow-up message. `model` is the short name it launched on (a
+ * follow-up's is its session's); `null` in journals from before #600. The
+ * exact version is in the turn's `agent_session`. `model_reason` is why it
+ * launched on that model (#603; a follow-up's is its session's); `null` in
+ * journals from before it.
+ */
+export const AgentStartedEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('agent_started'),
+    content: z.object({
+        role: AgentRoleSchema,
+        prompt: z.string(),
+        follow_up_of: z.string().nullable().default(null),
+        model: z.string().nullable().default(null),
+        model_reason: ModelReasonSchema.nullable().default(null),
+    }),
+})
+
+/** The session an agent turn ran in; `null` in journals from before #363. */
+const SESSION_FIELDS = { session_id: z.string().nullable().default(null) }
+
+/**
+ * An agent finished with a result that fits its role's schema. One option
+ * per role: keep it in step with `RoleResultSchema` (a test checks every
+ * role parses).
+ */
+export const AgentFinishedEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('agent_finished'),
+    content: z.discriminatedUnion('role', [
+        RoleResultSchema.options[0].extend(SESSION_FIELDS),
+        RoleResultSchema.options[1].extend(SESSION_FIELDS),
+        RoleResultSchema.options[2].extend(SESSION_FIELDS),
+        RoleResultSchema.options[3].extend(SESSION_FIELDS),
+        RoleResultSchema.options[4].extend(SESSION_FIELDS),
+        RoleResultSchema.options[5].extend(SESSION_FIELDS),
+        RoleResultSchema.options[6].extend(SESSION_FIELDS),
+        RoleResultSchema.options[7].extend(SESSION_FIELDS),
+        RoleResultSchema.options[8].extend(SESSION_FIELDS),
+    ]),
+})
+
+/**
+ * How an agent's turn failed:
+ * - `agent`: the turn itself failed (an error result, a timeout).
+ * - `result`: no structured output, or output that misfits the role's schema.
+ * - `guard`: the after-turn check found a change the role may not make.
+ * - `engine`: the engine's side failed (the SDK crashed, or a follow-up's
+ *   session is gone).
+ * - `evidence`: a test-writer's `already_done` evidence did not check out
+ *   (#495). Replay makes it from a failed `already_done_checked`, not from
+ *   an `agent_failed` record.
+ * - `chore`: a chore the writer's result asked the engine for could not be
+ *   done (#618): a file mode on a path the role may not write, an unknown
+ *   generator, or a failed one. See `chores_done`.
+ *
+ * The decision step counts all but `engine` as failed tries; an engine
+ * failure starts a fresh agent without using one.
+ */
+export const AgentFailureSchema = z.enum([
+    'agent',
+    'result',
+    'guard',
+    'engine',
+    'evidence',
+    'chore',
+])
+
+export type AgentFailure = z.infer<typeof AgentFailureSchema>
+
+/**
+ * An agent's turn failed, journaled once per turn with how it failed.
+ * `session_gone`: a follow-up found its session closed or gone (#521), so no
+ * agent ran. Replay forgets the session as a closed one, and counts no engine
+ * failure; the fix round goes to a fresh agent. `false` in older journals.
+ * `out_of_turns`: the turn reached its role's turn limit (#543), as the
+ * launcher said; the PR lists such tickets. `false` in older journals.
+ */
+export const AgentFailedEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('agent_failed'),
+    content: z.object({
+        role: AgentRoleSchema,
+        error: z.string(),
+        failure: AgentFailureSchema.default('agent'),
+        ...SESSION_FIELDS,
+        session_gone: z.boolean().default(false),
+        out_of_turns: z.boolean().default(false),
+    }),
+})
+
+/**
+ * The engine did the chores a writer's result asked for (#618), after its
+ * turn and before the checks: `file_modes` it set, and `checks`, one
+ * `generate` check per named generator it ran. `problems` lists what it
+ * could not do (a path the role may not write, an unknown generator, a
+ * failed one); with any, an `agent_failed` (`chore`) follows, for the
+ * role's fix loop.
+ */
+export const ChoresDoneEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('chores_done'),
+    content: z.object({
+        role: AgentRoleSchema,
+        file_modes: z.array(FileModeSchema),
+        checks: z.array(GateCheckSchema),
+        problems: z.array(z.string()),
+    }),
+})
+
+/** The launcher's summary of one agent turn's model session. */
+export const AgentSessionEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('agent_session'),
+    content: z.object({ role: AgentRoleSchema, session: AgentSessionSchema }),
+})
+
+/**
+ * The engine closed an agent's session: nothing can send it a follow-up
+ * anymore (its step's result was accepted, or its step ended another way).
+ * The decision step never follows up a closed session, even one a later
+ * record names again; a fix round that would have gone to it goes to a
+ * fresh agent.
+ */
+export const AgentSessionClosedEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('agent_session_closed'),
+    content: z.object({ role: AgentRoleSchema, session_id: z.string() }),
+})
+
+/**
+ * Shared `.git` files changed during an agent's turn by something the
+ * engine can't attribute to the agent: config outside the ticket branch's
+ * own section, `info/exclude`, or hooks, such as another agent's
+ * `git push -u` in the same repo. The change is neither blamed on the agent
+ * nor undone; this record only notes it.
+ */
+export const SharedGitChangedEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('shared_git_changed'),
+    content: z.object({
+        role: AgentRoleSchema,
+        /** What changed, such as `the shared .git/config changed`. */
+        changes: z.array(z.string()),
+    }),
+})

package/engine/journal/agent-message-entries.ts

@@ -0,0 +1,72 @@
+import { z } from 'zod'
+
+import { ENTRY_FIELDS } from './entry-fields'
+
+/** The journal entries of agent messages: each one sent, and each delivery. */
+
+/**
+ * What an agent message did when it was sent:
+ * - `queued`: it waits for its recipients' next tool calls.
+ * - `not_delivered`: nobody can get it (the receiver's ticket is over, or
+ *   `all` found no one); it stays in the journal.
+ * - `refused`: it broke a rule (a reviewer, a bad address, the cap).
+ */
+export const AgentMessageStatusSchema = z.enum([
+    'queued',
+    'not_delivered',
+    'refused',
+])
+
+export type AgentMessageStatus = z.infer<typeof AgentMessageStatusSchema>
+
+/**
+ * One agent message, word for word, as it was sent. Addresses look like
+ * `implementer#11`: a role and a ticket.
+ */
+export const AgentMessageSchema = z.object({
+    /** `msg-<n>`: n is 1 + the earlier `agent_message` records in the journal. */
+    id: z.string().min(1),
+    /** The sender's address. */
+    from: z.string(),
+    /** What the sender asked for: an address, or `all`. */
+    to: z.string(),
+    text: z.string(),
+    status: AgentMessageStatusSchema,
+    /** The addresses it waits for; empty unless queued. */
+    recipients: z.array(z.string()),
+    /** Why it was refused or not delivered; `null` when queued. */
+    reason: z.string().nullable(),
+})
+
+export type AgentMessage = z.infer<typeof AgentMessageSchema>
+
+/** An agent sent a message. `ticket` and `role` are the sender's. */
+export const AgentMessageEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('agent_message'),
+    content: AgentMessageSchema,
+})
+
+/** Messages handed to their receiver at one tool call, word for word. */
+export const AgentMessageDeliveredSchema = z.object({
+    /** The receiver's address. */
+    to: z.string(),
+    /** The ids of the messages handed over together. */
+    ids: z.array(z.string()),
+    /** The tool call they rode on. */
+    tool_name: z.string().nullable(),
+    /** The additional context the agent got, word for word. */
+    text: z.string(),
+})
+
+export type AgentMessageDelivered = z.infer<typeof AgentMessageDeliveredSchema>
+
+/**
+ * Agent messages reached their receiver. `ticket` and `role` are the
+ * receiver's; the record's time is when it saw them.
+ */
+export const AgentMessageDeliveredEntrySchema = z.object({
+    ...ENTRY_FIELDS,
+    kind: z.literal('agent_message_delivered'),
+    content: AgentMessageDeliveredSchema,
+})