@alecsibilia/luca 14.0.0-alpha.2 → 14.0.0-alpha.21

package/engine/intake/intake-checks.ts

@@ -2,10 +2,12 @@ import groupBy from 'lodash/groupBy'
 import sortBy from 'lodash/sortBy'
 import uniq from 'lodash/uniq'
 
+import { criteriaProblems } from './criteria-checks'
 import type {
     IntakeOutcome,
     IntakeProblem,
     IntakeRead,
+    LeftOutTicket,
     TicketSnapshot,
 } from './intake-schemas'
 
@@ -15,6 +17,7 @@ import {
     type EngineConfig,
 } from '../config/engine-config'
 import {
+    HUMAN_LABEL,
     READY_LABEL,
     REFACTOR_LABEL,
     RELEASE_LABEL_PREFIX,
@@ -94,6 +97,88 @@ export const outsideBlockerNumbers = ({
 
 type Finding = { ticket: number | null; message: string }
 
+/** Whether a ticket is for a person (#499): it has `ready-for-human`. */
+export const isForAPerson = ({ issue }: { issue: TrackerIssue }): boolean =>
+    issue.labels.includes(HUMAN_LABEL)
+
+/**
+ * The open tickets intake leaves out of the run (#499), by number: each one
+ * for a person, and each other one blocked by one, directly or through
+ * other open tickets of the spec. A ticket for a person stops the walk: what
+ * blocks it doesn't matter to the tickets that wait on it. Pure.
+ *
+ * @example
+ * leftOutTickets({ open: [agent11, human12, agent13BlockedBy12] })
+ * // [{ number: 12, reason: 'for_a_person', ... },
+ * //  { number: 13, reason: 'waits_on_person', waits_on: [12], through: [], ... }]
+ */
+export const leftOutTickets = ({
+    open,
+}: {
+    /** The spec's open sub-tickets. */
+    open: TrackerIssue[]
+}): LeftOutTicket[] => {
+    const byNumber = new Map(open.map((issue) => [issue.number, issue]))
+    const blockersOf = (issue: TrackerIssue): TrackerIssue[] =>
+        blockerNumbers({ issue }).flatMap((number) => {
+            const blocker = byNumber.get(number)
+            return blocker === undefined ? [] : [blocker]
+        })
+    const found = new Map<number, number[]>()
+    // The tickets for a person that `issue` waits on. A blocker loop among
+    // agent tickets refuses the run, so a loop just stops the walk here.
+    const waitsOn = (issue: TrackerIssue, walking: number[]): number[] => {
+        const known = found.get(issue.number)
+        if (known !== undefined) return known
+        if (walking.includes(issue.number)) return []
+        const people = sortBy(
+            uniq(
+                blockersOf(issue).flatMap((blocker) =>
+                    isForAPerson({ issue: blocker })
+                        ? [blocker.number]
+                        : waitsOn(blocker, [...walking, issue.number])
+                )
+            )
+        )
+        found.set(issue.number, people)
+        return people
+    }
+    return sortBy(open, 'number').flatMap((issue): LeftOutTicket[] => {
+        const { number, title, url } = issue
+        if (isForAPerson({ issue })) {
+            return [
+                {
+                    number,
+                    title,
+                    url,
+                    reason: 'for_a_person',
+                    waits_on: [],
+                    through: [],
+                },
+            ]
+        }
+        const waits_on = waitsOn(issue, [])
+        if (waits_on.length === 0) return []
+        const through = blockersOf(issue)
+            .filter(
+                (blocker) =>
+                    !isForAPerson({ issue: blocker }) &&
+                    waitsOn(blocker, []).length > 0
+            )
+            .map((blocker) => blocker.number)
+        return [
+            {
+                number,
+                title,
+                url,
+                reason: 'waits_on_person',
+                waits_on,
+                through: sortBy(through),
+            },
+        ]
+    })
+}
+
 /**
  * Walks blockers depth-first, blockers before the tickets they block. Returns
  * that order and every loop found, as a path such as `[12, 13, 12]`.
@@ -142,14 +227,61 @@ const groupFindings = ({
     }))
 }
 
+/**
+ * Intake's checks on the engine config, as messages for a person: a test
+ * command, and one with bun's per-test results unless every ticket in
+ * `tickets` is a refactor ticket (the red check reads per-test results,
+ * which only bun's give today). Pure. Empty when the config is fine.
+ *
+ * Intake runs it on the open tickets agents build; a resume runs it on the
+ * repo's config file before taking its new build fields (#516).
+ *
+ * @example
+ * configProblems({ config: { ...config, checks: {} }, tickets: [] })
+ * // ['The engine config (.luca/config.json) has no test command at checks.test.']
+ */
+export const configProblems = ({
+    config,
+    tickets,
+}: {
+    config: EngineConfig
+    tickets: { labels: string[] }[]
+}): string[] => {
+    const tests = testCommands({ config })
+    if (tests.length === 0) {
+        return [
+            `The engine config (${ENGINE_CONFIG_FILE}) has no test command at checks.test.`,
+        ]
+    }
+    if (
+        !tests.some(({ results }) => results === 'bun') &&
+        tickets.some((ticket) => !ticket.labels.includes(REFACTOR_LABEL))
+    ) {
+        return [
+            `The engine config (${ENGINE_CONFIG_FILE}) has no test command with bun results at checks.test, so the red check can't prove new tests fail first. Add a \`bun test\` command, or label every open ticket \`${REFACTOR_LABEL}\`.`,
+        ]
+    }
+    return []
+}
+
 /**
  * Intake's checks: is the spec, and every open ticket in it, ready to build?
  *
  * Pure. Every miss refuses the whole run, and every problem is collected (not
  * just the first), grouped by the spec or ticket it is on. Config and spec
- * problems refuse even an empty spec; otherwise a spec with no open tickets
- * has nothing to do. When everything passes, the result is the snapshot the
- * run builds from, with tickets in blocker order.
+ * problems refuse even an empty spec; otherwise a spec with nothing left to
+ * build has nothing to do. When everything passes, the result is the
+ * snapshot the run builds from, with tickets in blocker order.
+ *
+ * Tickets for a person (#499): an open ticket with `ready-for-human` (even
+ * with `ready-for-agent` too) is left out of the run, unchecked. An agent
+ * ticket that waits on one, directly or through other tickets, is left out
+ * too, but still checked, so a later run can build it. The run builds the
+ * rest. A ticket with neither ready label is still refused.
+ *
+ * Each open agent ticket's acceptance criteria must hold no placeholders, no
+ * untestable asks (#541), and nothing about behaviour that doesn't change
+ * (#653), per `criteriaProblems`; the spec's text isn't checked.
  */
 export const checkIntake = ({
     config,
@@ -160,28 +292,19 @@ export const checkIntake = ({
 }): IntakeOutcome => {
     const { spec, sub_tickets, outside_blockers } = intake_read
     const findings: Finding[] = []
+    const open = sortBy(
+        sub_tickets.filter((ticket) => ticket.state === 'open'),
+        'number'
+    )
+    const left_out = leftOutTickets({ open })
+    const leftOut = new Set(left_out.map(({ number }) => number))
+    // Every open ticket an agent builds: in this run, or once a person is done.
+    const forAgents = open.filter((issue) => !isForAPerson({ issue }))
     const onSpec = (message: string) =>
         findings.push({ ticket: spec.number, message })
 
-    const tests = testCommands({ config })
-    if (tests.length === 0) {
-        findings.push({
-            ticket: null,
-            message: `The engine config (${ENGINE_CONFIG_FILE}) has no test command at checks.test.`,
-        })
-    } else if (
-        !tests.some(({ results }) => results === 'bun') &&
-        sub_tickets.some(
-            (ticket) =>
-                ticket.state === 'open' &&
-                !ticket.labels.includes(REFACTOR_LABEL)
-        )
-    ) {
-        // The red check reads per-test results, which only bun's give today.
-        findings.push({
-            ticket: null,
-            message: `The engine config (${ENGINE_CONFIG_FILE}) has no test command with bun results at checks.test, so the red check can't prove new tests fail first. Add a \`bun test\` command, or label every open ticket \`${REFACTOR_LABEL}\`.`,
-        })
+    for (const message of configProblems({ config, tickets: forAgents })) {
+        findings.push({ ticket: null, message })
     }
     if (spec.state !== 'open') onSpec('The spec is closed.')
     if (section({ body: spec.body, heading: 'Testing Decisions' }) === '') {
@@ -196,10 +319,6 @@ export const checkIntake = ({
         )
     }
 
-    const open = sortBy(
-        sub_tickets.filter((ticket) => ticket.state === 'open'),
-        'number'
-    )
     const closedTickets = sortBy(
         sub_tickets
             .filter((ticket) => ticket.state === 'closed')
@@ -211,7 +330,7 @@ export const checkIntake = ({
         outside_blockers.map((issue) => [issue.number, issue])
     )
 
-    const tickets: TicketSnapshot[] = open.map((ticket) => {
+    const tickets: TicketSnapshot[] = forAgents.map((ticket) => {
         const onTicket = (message: string) =>
             findings.push({ ticket: ticket.number, message })
         if (section({ body: ticket.body, heading: 'What to build' }) === '') {
@@ -228,6 +347,12 @@ export const checkIntake = ({
         if (criteria.length === 0) {
             onTicket('The ticket has no checkbox under "Acceptance criteria".')
         }
+        for (const message of criteriaProblems({
+            ticket: ticket.number,
+            criteria,
+        })) {
+            onTicket(message)
+        }
         if (!ticket.labels.includes(READY_LABEL)) {
             onTicket(`The ticket does not have the ${READY_LABEL} label.`)
         }
@@ -273,10 +398,18 @@ export const checkIntake = ({
     if (findings.length > 0) {
         return { outcome: 'refused', problems: groupFindings({ findings }) }
     }
-    if (tickets.length === 0) {
-        return { outcome: 'nothing_to_do', closed_tickets: closedTickets }
+    const byNumber = new Map(
+        tickets
+            .filter((ticket) => !leftOut.has(ticket.number))
+            .map((ticket) => [ticket.number, ticket])
+    )
+    if (byNumber.size === 0) {
+        return {
+            outcome: 'nothing_to_do',
+            closed_tickets: closedTickets,
+            left_out,
+        }
     }
-    const byNumber = new Map(tickets.map((ticket) => [ticket.number, ticket]))
     return {
         outcome: 'passed',
         snapshot: {
@@ -290,6 +423,7 @@ export const checkIntake = ({
             },
             tickets: order.flatMap((number) => byNumber.get(number) ?? []),
             closed_tickets: closedTickets,
+            left_out,
         },
     }
 }

package/engine/intake/intake-schemas.ts

@@ -62,11 +62,41 @@ export const TicketSnapshotSchema = z.object({
 
 export type TicketSnapshot = z.infer<typeof TicketSnapshotSchema>
 
-/** The spec and its open tickets, in blocker order (blockers first). */
+/**
+ * Why intake left an open ticket out of the run (#499): `for_a_person`, as
+ * it has the `ready-for-human` label, or `waits_on_person`, as it is
+ * blocked, directly or through other tickets, by an open one that has.
+ */
+export const LeftOutReasonSchema = z.enum(['for_a_person', 'waits_on_person'])
+
+export type LeftOutReason = z.infer<typeof LeftOutReasonSchema>
+
+/** An open ticket intake left out of the run, and why (#499). */
+export const LeftOutTicketSchema = z.object({
+    number: z.number().int().positive(),
+    title: z.string(),
+    url: z.string().default(''),
+    reason: LeftOutReasonSchema,
+    /** For `waits_on_person`: the tickets for a person it waits on. */
+    waits_on: z.array(z.number().int().positive()).default([]),
+    /**
+     * For `waits_on_person`: its own blockers that wait on a person too,
+     * when it waits through them rather than (or as well as) directly.
+     */
+    through: z.array(z.number().int().positive()).default([]),
+})
+
+export type LeftOutTicket = z.infer<typeof LeftOutTicketSchema>
+
+/**
+ * The spec and its open tickets, in blocker order (blockers first), and
+ * the open tickets left out for a person (#499), by number.
+ */
 export const IntakeSnapshotSchema = z.object({
     spec: SpecSnapshotSchema,
     tickets: z.array(TicketSnapshotSchema),
     closed_tickets: z.array(z.number().int().positive()),
+    left_out: z.array(LeftOutTicketSchema).default([]),
 })
 
 export type IntakeSnapshot = z.infer<typeof IntakeSnapshotSchema>
@@ -74,5 +104,10 @@ export type IntakeSnapshot = z.infer<typeof IntakeSnapshotSchema>
 /** The result of intake's checks. */
 export type IntakeOutcome =
     | { outcome: 'refused'; problems: IntakeProblem[] }
-    | { outcome: 'nothing_to_do'; closed_tickets: number[] }
+    | {
+          outcome: 'nothing_to_do'
+          closed_tickets: number[]
+          /** Open tickets left out for a person (#499): none left to build. */
+          left_out: LeftOutTicket[]
+      }
     | { outcome: 'passed'; snapshot: IntakeSnapshot }

package/engine/jev/clef-client.ts

@@ -0,0 +1,259 @@
+import mapValues from 'lodash/mapValues'
+import { z } from 'zod'
+
+import type { JevAnswer, JevFailureReason, JevRequest } from './jev-schemas'
+
+/** Workers AI's REST API: `<base>/<account id>/ai/run/<model id>`. */
+const WORKERS_AI_ACCOUNTS_URL = 'https://api.cloudflare.com/client/v4/accounts'
+
+/** How much of an error body a failed call keeps. */
+const ERROR_SNIPPET_CHARS = 300
+
+/**
+ * The decision model's answers by question id, and the model that gave
+ * them; or why there are none. Never a throw.
+ */
+export type DecisionModelReply =
+    | {
+          ok: true
+          answers: Record<string, JevAnswer>
+          /** The model that answered; left out, the client's own. */
+          model?: string
+      }
+    | { ok: false; reason: JevFailureReason; error: string }
+
+/** Asks the decision model one request. The signal cuts the call short. */
+export type DecisionModelClient = {
+    /** The model it asks, such as `@cf/cloudflare/clef`; journaled on each ask. */
+    model: string
+    ask: (args: {
+        request: JevRequest
+        signal: AbortSignal
+    }) => Promise<DecisionModelReply>
+}
+
+/** The part of `fetch` the client uses, so tests can hand in a fake. */
+export type DecisionModelFetch = (
+    url: string,
+    init: RequestInit
+) => Promise<Response>
+
+/** The Cloudflare account and Workers AI token the client calls with. */
+export type ClefCredentials = { account_id: string; api_token: string }
+
+/**
+ * Clef's output: its answers by question id, and the model's short name.
+ * Workers AI wraps it as `{ result, success, errors, messages }`, so it is
+ * read inside `result` or bare.
+ */
+const ClefOutputSchema = z.object({
+    model: z.string().min(1).nullish().catch(null),
+    answers: z.record(z.string(), z.unknown()),
+})
+
+const EnvelopeSchema = z.object({ result: ClefOutputSchema })
+
+/** A Cloudflare v4 error body: `{ success: false, errors: [{ code, message }] }`. */
+const ErrorEnvelopeSchema = z.object({
+    errors: z.array(z.object({ message: z.string() })).min(1),
+})
+
+/**
+ * The answer fields the engine knows: Clef's `choice` (a criteria key),
+ * `score` (a place on the scale), and `noul` (the probability of yes), each
+ * with a `confidence` except noul. Jev's `probability` and a plain `value`
+ * read too. A field that doesn't fit falls back to `null`.
+ */
+const LooseAnswerSchema = z.object({
+    choice: z.string().nullish().catch(null),
+    score: z.union([z.string(), z.number()]).nullish().catch(null),
+    noul: z.number().nullish().catch(null),
+    probability: z.number().nullish().catch(null),
+    value: z.union([z.string(), z.number()]).nullish().catch(null),
+    confidence: z.number().nullish().catch(null),
+})
+
+const normalizeAnswer = (raw: unknown): JevAnswer => {
+    const parsed = LooseAnswerSchema.safeParse(raw)
+    if (!parsed.success) return { value: null, confidence: null, raw }
+    const { choice, score, noul, probability, value, confidence } = parsed.data
+    return {
+        value: choice ?? score ?? noul ?? probability ?? value ?? null,
+        confidence: confidence ?? null,
+        raw,
+    }
+}
+
+const isAbort = ({
+    error,
+    signal,
+}: {
+    error: unknown
+    signal: AbortSignal
+}): boolean =>
+    signal.aborted ||
+    (error instanceof Error &&
+        (error.name === 'AbortError' || error.name === 'TimeoutError'))
+
+/** Removes the token from any text, so it never lands in the journal. */
+const hideToken = ({ text, token }: { text: string; token: string }): string =>
+    text.split(token).join('[token]')
+
+const parseJson = (text: string): unknown => {
+    try {
+        return JSON.parse(text)
+    } catch {
+        return undefined
+    }
+}
+
+/** What a failed call's body says: Cloudflare's error messages, or its start. */
+const errorBodyText = (text: string): string => {
+    const envelope = ErrorEnvelopeSchema.safeParse(parseJson(text))
+    return envelope.success
+        ? envelope.data.errors.map(({ message }) => message).join('; ')
+        : text.slice(0, ERROR_SNIPPET_CHARS)
+}
+
+/**
+ * The URL that runs `model` on Workers AI for `account_id`. The model id's
+ * `@` and `/` stay as they are, as Cloudflare's docs write them. Pure.
+ *
+ * @example
+ * clefUrl({ account_id: 'abc', model: '@cf/cloudflare/clef' })
+ * // 'https://api.cloudflare.com/client/v4/accounts/abc/ai/run/@cf/cloudflare/clef'
+ */
+export const clefUrl = ({
+    account_id,
+    model,
+}: {
+    account_id: string
+    model: string
+}): string =>
+    `${WORKERS_AI_ACCOUNTS_URL}/${encodeURIComponent(account_id)}/ai/run/${model}`
+
+/** The name Clef wants in the body: the model id's last part, `clef` or `clef-flash`. */
+const shortName = (model: string): string => model.split('/').at(-1) ?? model
+
+/**
+ * The decision model client for Cloudflare's **Clef** on Workers AI (#534).
+ * It never throws: missing credentials, a timeout, a bad status, or a bad
+ * body each come back as a failed reply, and no error text ever holds the
+ * token. HTTP 401 and 403 are `rejected` (the token is wrong or lacks Workers
+ * AI), which turns the decision model off for the run; anything else is an
+ * `error` or a `timeout`, and the next ask tries again.
+ *
+ * @param credentials - The account id and token (see
+ *   `loadDecisionModelCredentials`); `null` or an empty one is
+ *   `missing_credentials`, and then `fetch` is never called.
+ * @param model - The Workers AI model id, such as `@cf/cloudflare/clef`.
+ * @param fetch - Defaults to the global `fetch`.
+ *
+ * @example
+ * const clef = createClefClient({ credentials: { account_id, api_token }, model: '@cf/cloudflare/clef' })
+ * const reply = await clef.ask({ request, signal: AbortSignal.timeout(10_000) })
+ * if (reply.ok) console.log(reply.answers)
+ */
+export const createClefClient = ({
+    credentials,
+    model,
+    fetch: fetchClef,
+}: {
+    credentials: ClefCredentials | null
+    model: string
+    fetch?: DecisionModelFetch
+}): DecisionModelClient => ({
+    model,
+    ask: async ({ request, signal }) => {
+        if (
+            credentials === null ||
+            credentials.account_id === '' ||
+            credentials.api_token === ''
+        ) {
+            return {
+                ok: false,
+                reason: 'missing_credentials',
+                error: 'No Cloudflare account id or token is set.',
+            }
+        }
+        const token = credentials.api_token
+        try {
+            const response = await (fetchClef ?? fetch)(
+                clefUrl({ account_id: credentials.account_id, model }),
+                {
+                    method: 'POST',
+                    headers: {
+                        authorization: `Bearer ${token}`,
+                        'content-type': 'application/json',
+                    },
+                    body: JSON.stringify({
+                        model: shortName(model),
+                        state: request.state,
+                        questions: request.questions,
+                    }),
+                    signal,
+                }
+            )
+            const text = await response.text()
+            if (!response.ok) {
+                const said = hideToken({ text: errorBodyText(text), token })
+                const rejected =
+                    response.status === 401 || response.status === 403
+                return rejected
+                    ? {
+                          ok: false,
+                          reason: 'rejected',
+                          error: `Cloudflare turned the token down (HTTP ${response.status}): ${said}`,
+                      }
+                    : {
+                          ok: false,
+                          reason: 'error',
+                          error: `Clef answered ${response.status}: ${said}`,
+                      }
+            }
+            const json = parseJson(text)
+            if (json === undefined) {
+                return {
+                    ok: false,
+                    reason: 'error',
+                    error: 'Clef answered with a body that is not JSON.',
+                }
+            }
+            const wrapped = EnvelopeSchema.safeParse(json)
+            const output = wrapped.success
+                ? { success: true as const, data: wrapped.data.result }
+                : ClefOutputSchema.safeParse(json)
+            if (!output.success) {
+                return {
+                    ok: false,
+                    reason: 'error',
+                    error: hideToken({
+                        text: `Clef answered with no answers: ${z.prettifyError(output.error)}`,
+                        token,
+                    }),
+                }
+            }
+            return {
+                ok: true,
+                model: output.data.model ?? model,
+                answers: mapValues(output.data.answers, normalizeAnswer),
+            }
+        } catch (error) {
+            if (isAbort({ error, signal })) {
+                return {
+                    ok: false,
+                    reason: 'timeout',
+                    error: 'Clef did not answer in time.',
+                }
+            }
+            return {
+                ok: false,
+                reason: 'error',
+                error: hideToken({
+                    text: `Clef call failed: ${String(error)}`,
+                    token,
+                }),
+            }
+        }
+    },
+})