@alecsibilia/luca 14.0.0-alpha.2 → 14.0.0-alpha.21

package/engine/jev/jev-shadow-helpers.ts

@@ -0,0 +1,218 @@
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+import { afterAll, beforeAll, expect } from 'bun:test'
+
+import type { DecisionModelClient } from './clef-client'
+import type { JevAnswer, JevQuestion, JevRequest } from './jev-schemas'
+
+import type { ScriptedTurn } from '../agents/scripted-launcher'
+import type { JournalRecord } from '../journal/journal-record'
+import { runPractice, type PracticeRun } from '../testing/practice-repo'
+
+/**
+ * Helpers shared by the end-to-end shadow-mode tests
+ * (`jev-shadow.e2e.test.ts` and `jev-shadow-findings.e2e.test.ts`): the fake
+ * decision model that disagrees with everything, and the run it is compared
+ * against.
+ */
+
+/** A shadow-mode record: an ask, its reply, or the decision model turning off. */
+export const isJev = (record: JournalRecord): boolean =>
+    record.kind.startsWith('jev_') || record.kind === 'decision_model_off'
+
+/**
+ * Launches with test-run timings (" [7.00ms]") removed and commit shas
+ * (which carry the time) masked: a fix-loop follow-up quotes test output,
+ * whose timings differ between two runs, and bun leaves the timing off a
+ * test that runs very fast, so a timing is dropped whether or not it's there.
+ */
+const withoutTimings = (launches: PracticeRun['launches']) =>
+    launches.map((launch) => ({
+        ...launch,
+        prompt: launch.prompt
+            .replace(/ ?\[\d+(\.\d+)?m?s\]/g, '')
+            .replace(/\b[0-9a-f]{40}\b/g, '[sha]'),
+    }))
+
+/**
+ * Record kinds in order, but the final review's lenses run at once, so
+ * the kinds between its start and its pass are sorted.
+ */
+const kindsInOrder = (records: JournalRecord[]): string[] => {
+    const kinds = records
+        .filter((record) => !isJev(record))
+        .map(({ kind }) => kind)
+    const from = kinds.indexOf('final_review_started')
+    const to = kinds.indexOf('final_review_passed')
+    if (from === -1 || to === -1) return kinds
+    return [
+        ...kinds.slice(0, from + 1),
+        ...kinds.slice(from + 1, to).toSorted(),
+        ...kinds.slice(to),
+    ]
+}
+
+/**
+ * Launches in order, but the final review's lenses (which start at once, in
+ * any order) sorted by role, with their session numbers masked.
+ */
+const launchesInOrder = (launches: PracticeRun['launches']) => {
+    const lens = (launch: PracticeRun['launches'][number]) =>
+        launch.role.endsWith('-lens')
+    return [
+        ...launches.filter((launch) => !lens(launch)),
+        ...launches
+            .filter(lens)
+            .map((launch) => ({
+                ...launch,
+                session_id: launch.session_id.replace(/-\d+$/, '-[n]'),
+            }))
+            .toSorted((left, right) => left.role.localeCompare(right.role)),
+    ]
+}
+
+/** The run as it would look without Jev: the same steps, prompts, and PR. */
+export const expectSameRun = ({
+    run,
+    without,
+}: {
+    run: PracticeRun
+    without: PracticeRun
+}) => {
+    expect(run.action).toEqual(without.action)
+    expect(kindsInOrder(run.records)).toEqual(kindsInOrder(without.records))
+    expect(withoutTimings(launchesInOrder(run.launches))).toEqual(
+        withoutTimings(launchesInOrder(without.launches))
+    )
+    expect(run.tracker.pullRequests()).toEqual(without.tracker.pullRequests())
+}
+
+/** The model id the fake decision models say they are. */
+export const FAKE_MODEL = '@cf/cloudflare/clef'
+
+/**
+ * A Jev that answers every question, and never as the engine would: the
+ * last option of each choice, the top of each score, yes to every skill.
+ */
+const disagreeingAnswer = (question: JevQuestion): JevAnswer => {
+    switch (question.type) {
+        case 'choice': {
+            const choice = Object.keys(question.criteria).at(-1) ?? null
+            return {
+                value: choice,
+                confidence: 0.7,
+                raw: { choice, confidence: 0.7 },
+            }
+        }
+        case 'score': {
+            const score = question.criteria.at(-1) ?? null
+            return {
+                value: score,
+                confidence: 0.6,
+                raw: { score, confidence: 0.6 },
+            }
+        }
+        case 'noul':
+            return { value: 0.95, confidence: null, raw: { probability: 0.95 } }
+    }
+}
+
+/** Whether a request has a choice Clef refuses: fewer than two options (#546). */
+export const hasOneOptionChoice = (request: JevRequest): boolean =>
+    Object.values(request.questions).some(
+        (question) =>
+            question.type === 'choice' &&
+            Object.keys(question.criteria).length < 2
+    )
+
+const disagreeingJev = (): DecisionModelClient & {
+    signals: AbortSignal[]
+    requests: JevRequest[]
+} => {
+    const signals: AbortSignal[] = []
+    const requests: JevRequest[] = []
+    return {
+        signals,
+        requests,
+        model: FAKE_MODEL,
+        ask: async ({ request, signal }) => {
+            signals.push(signal)
+            requests.push(request)
+            return {
+                ok: true,
+                answers: Object.fromEntries(
+                    Object.entries(request.questions).map(([id, question]) => [
+                        id,
+                        disagreeingAnswer(question),
+                    ])
+                ),
+            }
+        },
+    }
+}
+
+type Asked = Extract<JournalRecord, { kind: 'jev_asked' }>
+type Answered = Extract<JournalRecord, { kind: 'jev_answered' }>
+type Failed = Extract<JournalRecord, { kind: 'jev_failed' }>
+
+export const asked = (records: JournalRecord[]): Asked[] =>
+    records.filter((record): record is Asked => record.kind === 'jev_asked')
+
+/** The record right after each ask: its answer or its failure. */
+export const replyTo = (
+    records: JournalRecord[],
+    ask: Asked
+): Answered | Failed | undefined => {
+    const next = records[records.indexOf(ask) + 1]
+    return next?.kind === 'jev_answered' || next?.kind === 'jev_failed'
+        ? next
+        : undefined
+}
+
+export const SKILL_IDS = [
+    'tdd',
+    'systematic_debugging',
+    'codebase_design',
+    'domain_modeling',
+]
+
+/** A practice run with Jev, and the same run without it, built once. */
+export const shareRuns = ({ turns }: { turns: () => ScriptedTurn[] }) => {
+    const shared: {
+        roots: string[]
+        jev?: ReturnType<typeof disagreeingJev>
+        run?: PracticeRun
+        without?: PracticeRun
+    } = { roots: [] }
+    beforeAll(async () => {
+        const root = await mkdtemp(join(tmpdir(), 'luca-engine-jev-'))
+        shared.roots.push(root)
+        const jev = disagreeingJev()
+        shared.jev = jev
+        shared.run = await runPractice({
+            root: join(root, 'with-jev'),
+            turns: turns(),
+            jev: { client: jev },
+        })
+        shared.without = await runPractice({
+            root: join(root, 'without-jev'),
+            turns: turns(),
+        })
+    }, 120_000)
+    afterAll(async () => {
+        await Promise.all(
+            shared.roots.map((root) =>
+                rm(root, { recursive: true, force: true })
+            )
+        )
+    })
+    return () => {
+        const { jev, run, without } = shared
+        if (jev === undefined || run === undefined || without === undefined) {
+            throw new Error('The shared runs were not built.')
+        }
+        return { jev, run, without }
+    }
+}

package/engine/jev/jev-shadow.ts

@@ -1,12 +1,12 @@
 import { z } from 'zod'
 
-import type { JevClient, JevReply } from './jev-client'
+import type { DecisionModelClient, DecisionModelReply } from './clef-client'
 import type { JevAsk } from './jev-jobs'
-import { JevAnswerSchema } from './jev-schemas'
+import { ClefRequestSchema, JevAnswerSchema } from './jev-schemas'
 
 import type { Journal } from '../journal/journal'
 
-/** How long the engine waits for one Jev answer before it moves on. */
+/** How long the engine waits for one decision model answer before it moves on. */
 export const DEFAULT_JEV_TIMEOUT_MS = 10_000
 
 const JevAnswersSchema = z.record(z.string(), JevAnswerSchema)
@@ -15,26 +15,33 @@ const JevAnswersSchema = z.record(z.string(), JevAnswerSchema)
  * A reply whose answers don't fit the journal's schema (only a broken client
  * sends one) becomes an error reply, so appending it can never throw.
  */
-const checkedReply = (reply: JevReply): JevReply => {
+const checkedReply = (reply: DecisionModelReply): DecisionModelReply => {
     if (!reply.ok) return reply
     const parsed = JevAnswersSchema.safeParse(reply.answers)
-    if (parsed.success) return { ok: true, answers: parsed.data }
+    if (parsed.success) {
+        return { ok: true, answers: parsed.data, model: reply.model }
+    }
     return {
         ok: false,
         reason: 'error',
-        error: `Jev's answers do not fit their schema:\n${z.prettifyError(parsed.error)}`,
+        error: `The decision model's answers do not fit their schema:\n${z.prettifyError(parsed.error)}`,
     }
 }
 
-/** Jev and how long to wait for it, as `runEngine` takes them. */
+/**
+ * The decision model, how long to wait for it, and where to print the line
+ * when it turns off mid-run, as `runEngine` takes them.
+ */
 export type JevShadow = {
-    client: JevClient
+    client: DecisionModelClient
     /** Defaults to `DEFAULT_JEV_TIMEOUT_MS`. */
     timeout_ms?: number
+    /** Gets one plain line when the decision model turns off mid-run. */
+    log?: (line: string) => void
 }
 
 /**
- * Calls Jev once, and always comes back: a throw becomes an error reply, and
+ * Calls the decision model once, and always comes back: a throw becomes an error reply, and
  * a call past the timeout becomes a timeout reply even if the client ignores
  * the abort signal.
  */
@@ -43,19 +50,19 @@ const askWithTimeout = async ({
     ask,
     timeout_ms,
 }: {
-    jev: JevClient
+    jev: DecisionModelClient
     ask: JevAsk
     timeout_ms: number
-}): Promise<JevReply> => {
+}): Promise<DecisionModelReply> => {
     const controller = new AbortController()
     let timer: ReturnType<typeof setTimeout> | undefined
-    const timedOut = new Promise<JevReply>((resolve) => {
+    const timedOut = new Promise<DecisionModelReply>((resolve) => {
         timer = setTimeout(() => {
             controller.abort()
             resolve({
                 ok: false,
                 reason: 'timeout',
-                error: `Jev did not answer within ${timeout_ms} ms.`,
+                error: `The decision model did not answer within ${timeout_ms} ms.`,
             })
         }, timeout_ms)
     })
@@ -64,10 +71,10 @@ const askWithTimeout = async ({
             jev.ask({ request: ask.request, signal: controller.signal })
         )
         .catch(
-            (error: unknown): JevReply => ({
+            (error: unknown): DecisionModelReply => ({
                 ok: false,
                 reason: 'error',
-                error: `Jev threw: ${String(error)}`,
+                error: `The decision model threw: ${String(error)}`,
             })
         )
     try {
@@ -77,73 +84,111 @@ const askWithTimeout = async ({
     }
 }
 
+/** Asks the decision model about a step's asks, in shadow mode. */
+export type ShadowAsker = (asks: JevAsk[]) => Promise<void>
+
 /**
- * Asks Jev each question in **shadow mode**: journals `jev_asked` with the
- * engine's fixed choice, then `jev_answered` or `jev_failed`, linked back by
- * `asked_seq`. Jev's answers change nothing. A Jev error, timeout, or missing
- * key is journaled and never thrown, so the run goes on.
+ * Asks the decision model in **shadow mode** for one run: for each ask,
+ * journals `jev_asked` with the engine's fixed choice and the model, then
+ * `jev_answered` (with the model that answered) or `jev_failed`, linked back
+ * by `asked_seq`. The answers change nothing here; the launch step reads an
+ * `agent_model` answer whose ask is `acted` (#603). An ask outside Clef's limits
+ * (`ClefRequestSchema`) is skipped: not sent and not journaled. An error,
+ * timeout, or missing credentials is journaled and never thrown, so the run
+ * goes on.
+ *
+ * The first ask Cloudflare turns down (`rejected`: HTTP 401 or 403) turns
+ * the decision model **off** for the rest of the run: one
+ * `decision_model_off { reason: 'rejected' }` record and one line to `log`,
+ * and no ask after it is journaled or sent. The off switch lives in this
+ * asker, so each run (each `runEngine`) has its own. A timeout or an error
+ * doesn't turn it off; the next ask tries again.
  *
  * @example
- * await askJevInShadow({
- *     jev: createTypeSafeJev(),
- *     journal,
- *     asks: jevAsksBefore({ action, state }),
- * })
+ * const ask = shadowAsker({ jev: { client: createClefClient({ credentials, model }) }, journal })
+ * await ask(jevAsksBefore({ action, state }))
  */
-export const askJevInShadow = async ({
+export const shadowAsker = ({
     jev,
     journal,
-    asks,
-    timeout_ms,
 }: {
-    jev: JevClient
+    jev: JevShadow
     journal: Journal
-    asks: JevAsk[]
-    /** Defaults to `DEFAULT_JEV_TIMEOUT_MS`. */
-    timeout_ms?: number
-}): Promise<void> => {
-    for (const ask of asks) {
-        const { job, ticket, role, request, fixed } = ask
-        const asked = journal.append({
-            kind: 'jev_asked',
-            ticket,
-            role,
-            content: { job, request, fixed },
-        })
-        const started = performance.now()
-        const reply = checkedReply(
-            await askWithTimeout({
-                jev,
-                ask,
-                timeout_ms: timeout_ms ?? DEFAULT_JEV_TIMEOUT_MS,
+}): ShadowAsker => {
+    const { client, timeout_ms, log } = jev
+    let off = false
+    return async (asks) => {
+        for (const ask of asks) {
+            if (off) return
+            // Clef refuses it (a one-option choice, say), and the answer is
+            // obvious: no call and no record (#546).
+            if (!ClefRequestSchema.safeParse(ask.request).success) continue
+            const { job, ticket, role, request, fixed, acted } = ask
+            const asked = journal.append({
+                kind: 'jev_asked',
+                ticket,
+                role,
+                // Shadow mode unless the ask says its answer is acted on.
+                content: {
+                    job,
+                    request,
+                    fixed,
+                    model: client.model,
+                    acted: acted ?? false,
+                },
             })
-        )
-        const ms = Math.round(performance.now() - started)
-        if (reply.ok) {
+            const started = performance.now()
+            const reply = checkedReply(
+                await askWithTimeout({
+                    jev: client,
+                    ask,
+                    timeout_ms: timeout_ms ?? DEFAULT_JEV_TIMEOUT_MS,
+                })
+            )
+            const ms = Math.round(performance.now() - started)
+            if (reply.ok) {
+                journal.append({
+                    kind: 'jev_answered',
+                    ticket,
+                    role,
+                    content: {
+                        job,
+                        asked_seq: asked.seq,
+                        answers: reply.answers,
+                        ms,
+                        model: reply.model ?? client.model,
+                    },
+                })
+                continue
+            }
             journal.append({
-                kind: 'jev_answered',
+                kind: 'jev_failed',
                 ticket,
                 role,
                 content: {
                     job,
                     asked_seq: asked.seq,
-                    answers: reply.answers,
+                    reason: reply.reason,
+                    error: reply.error,
                     ms,
                 },
             })
-            continue
+            // Asks run at once on other tickets; only the first turns it off.
+            if (reply.reason === 'rejected' && !off) {
+                off = true
+                const detail = `${reply.error} No more asks this run.`
+                journal.append({
+                    kind: 'decision_model_off',
+                    ticket: null,
+                    role: null,
+                    content: {
+                        model: client.model,
+                        reason: 'rejected',
+                        detail,
+                    },
+                })
+                log?.(`decision model off: ${detail}`)
+            }
         }
-        journal.append({
-            kind: 'jev_failed',
-            ticket,
-            role,
-            content: {
-                job,
-                asked_seq: asked.seq,
-                reason: reply.reason,
-                error: reply.error,
-                ms,
-            },
-        })
     }
 }