@alecsibilia/luca 14.0.0-alpha.2 → 14.0.0-alpha.21

package/engine/gates/red-check.ts

@@ -1,6 +1,18 @@
 import type { RedCheckResult, TestCase, TestRun } from './gate-schemas'
+import { addedTestNames, testNamesIn } from './new-tests'
 
 import type { CriterionTests, TestRef } from '../agents/role-results'
+import { MAX_TESTS_PER_CRITERION } from '../core/loop-caps'
+
+/**
+ * A test file the ticket changed, as it was at the ticket's base and as it
+ * is now (`null`: not there).
+ */
+export type ChangedTestSource = {
+    file: string
+    base: string | null
+    current: string | null
+}
 
 const lastPart = (name: string): string =>
     (name.split(' > ').at(-1) ?? name).trim()
@@ -8,7 +20,44 @@ const lastPart = (name: string): string =>
 /** Quote-insensitive text, so a test name found in source matches its printed name. */
 const loose = (text: string): string => text.replace(/[\\'"`]/g, '')
 
-const findCase = ({
+/**
+ * Said after a new unmapped test found in the text of a test file that does
+ * not load yet: with no results to check against, a `test(` inside a string
+ * or a comment counts as a test too (#581).
+ */
+const NOT_LOADED_HINT =
+    '. The file does not load yet, so every test( in its text counts, even one inside a string or a comment'
+
+/**
+ * Whether a test name written in a file's text (`own`) is the test bun prints
+ * as `full_name`: equal to it, at its end after " > " (not split on the last
+ * " > ": a test name may hold one itself), or the same last part. One rule
+ * for a mapped name and for a name found in the text.
+ */
+const namesTest = ({
+    full_name,
+    own,
+}: {
+    full_name: string
+    own: string
+}): boolean => {
+    const full = full_name.trim()
+    return (
+        full === own ||
+        full.endsWith(` > ${own}`) ||
+        lastPart(full) === lastPart(own)
+    )
+}
+
+/**
+ * The test case a named test is, in its file: the one whose full name
+ * matches, else the only one whose last part does. `undefined` if none (or
+ * more than one) does.
+ *
+ * @example
+ * findCase({ cases: run.cases, test: { file: 'src/sum.test.ts', name: 'sum > adds' } })
+ */
+export const findCase = ({
     cases,
     test,
 }: {
@@ -67,16 +116,79 @@ const checkNewTest = ({
 }
 
 /**
- * The red check. Pure: proves every criterion has a test, every new test
- * fails, and every old test that passed before still passes.
+ * Every old test (passed at the baseline) still passes, in its file or
+ * moved (#609): one missing from its file but now in another test file
+ * under the same full name, a case the baseline didn't have there. Each
+ * such case stands for one old test only, so a name several old tests had
+ * counts as moved once.
+ */
+const checkOldTests = ({
+    baseline,
+    current,
+}: {
+    baseline: TestRun
+    current: TestRun
+}): { problems: string[]; notes: string[] } => {
+    const problems: string[] = []
+    const notes: string[] = []
+    const same = (a: TestCase, b: TestCase) =>
+        a.file === b.file && a.full_name === b.full_name
+    const movable = current.cases.filter(
+        (now) => !baseline.cases.some((old) => same(old, now))
+    )
+    const claimed = new Set<TestCase>()
+    for (const old of baseline.cases.filter(
+        ({ status }) => status === 'passed'
+    )) {
+        const label = `old test "${old.full_name}" in ${old.file}`
+        const now = current.cases.find((found) => same(found, old))
+        if (now !== undefined) {
+            if (now.status !== 'passed')
+                problems.push(`${label} no longer passes`)
+            continue
+        }
+        const moved = movable.find(
+            (found) => !claimed.has(found) && found.full_name === old.full_name
+        )
+        if (moved === undefined) {
+            problems.push(`${label} is missing`)
+            continue
+        }
+        claimed.add(moved)
+        const where = `old test "${old.full_name}" in ${moved.file} (moved from ${old.file})`
+        if (moved.status === 'passed')
+            notes.push(`${where}: still passes, as it should`)
+        else problems.push(`${where} no longer passes`)
+    }
+    return { problems, notes }
+}
+
+/**
+ * The red check. Pure: proves every criterion has a test, and at most
+ * `MAX_TESTS_PER_CRITERION`, every test the ticket added is mapped, every
+ * test file it created is in the feature map (when the repo has one), every
+ * new test fails, and every old test that passed before still passes. A
+ * test the ticket only moved to another test file, under the same full
+ * name, is an old test (#609): noted, not missing and not new.
  *
  * @param baseline - The test run before the test-writer started.
  * @param current - The test run after the test-writer finished.
  * @param test_files - The worktree's test files, per the engine config.
  * @param sources - The text of each mapped test file, `null` if missing.
+ * @param changed_test_sources - Each test file the ticket changed, as it was
+ *   at the ticket's base and as it is now: a new test is a name in the
+ *   second the first doesn't have (#581), and no changed or deleted test
+ *   file had at its base (a moved test, #609). For a file that has results in
+ *   `current`, only a name of a case that ran counts, so a `test(` inside a
+ *   string (a fixture) or a comment does not. A file that doesn't load yet
+ *   has none, so its text alone decides, and every `test(` in it counts.
+ * @param feature_map - The repo's feature map, when the config names one
+ *   (#583): its `path`, and `files`, the test files it lists (from
+ *   `featureMapFiles`), `null` when the map file is missing. A test file the
+ *   ticket created must be among them. Left out or `null`, nothing is checked.
  *
  * @example
- * const result = checkRed({ criteria_ids: ['AC1'], mapping, baseline, current, test_files, sources })
+ * const result = checkRed({ criteria_ids: ['AC1'], mapping, baseline, current, test_files, sources, changed_test_sources })
  * if (!result.ok) console.error(result.problems)
  */
 export const checkRed = ({
@@ -86,6 +198,8 @@ export const checkRed = ({
     current,
     test_files,
     sources,
+    changed_test_sources,
+    feature_map,
 }: {
     criteria_ids: string[]
     mapping: CriterionTests[]
@@ -93,6 +207,8 @@ export const checkRed = ({
     current: TestRun
     test_files: string[]
     sources: Record<string, string | null>
+    changed_test_sources: ChangedTestSource[]
+    feature_map?: { path: string; files: string[] | null } | null
 }): RedCheckResult => {
     const problems: string[] = []
     const notes: string[] = []
@@ -107,6 +223,67 @@ export const checkRed = ({
             notes.push(`unknown criterion id ${criterion_id}`)
         }
     }
+    for (const { criterion_id, tests } of mapping) {
+        if (tests.length > MAX_TESTS_PER_CRITERION) {
+            problems.push(
+                `${criterion_id} has ${tests.length} tests; write at most ${MAX_TESTS_PER_CRITERION} per criterion`
+            )
+        }
+    }
+    // A name some changed (or deleted) test file had at the base is a moved
+    // test, not a new one, whichever file it is in now (#609).
+    const old_names = new Set(
+        changed_test_sources.flatMap(({ base }) =>
+            base === null ? [] : testNamesIn({ source: base })
+        )
+    )
+    for (const { file, base, current: text } of changed_test_sources) {
+        const mapped = mapping
+            .flatMap(({ tests }) => tests)
+            .filter((test) => test.file === file)
+            .map(({ name }) => name.trim())
+        // A file that loaded has results: a name in its text is a new test
+        // only if a case of that name ran, so a `test('x'` inside a string
+        // (a fixture) or a comment is not one. A file with none (it doesn't
+        // load yet) is read from its text alone.
+        const loaded = !current.files_without_results.includes(file)
+        const ran = current.cases.filter((found) => found.file === file)
+        for (const name of addedTestNames({ base, current: text })) {
+            if (old_names.has(name)) continue
+            const own = name.trim()
+            if (
+                loaded &&
+                !ran.some(({ full_name }) => namesTest({ full_name, own }))
+            ) {
+                continue
+            }
+            if (!mapped.some((full) => namesTest({ full_name: full, own }))) {
+                problems.push(
+                    `"${name}" in ${file}: a new test the map doesn't name (map it to a criterion, or remove it)${loaded ? '' : NOT_LOADED_HINT}`
+                )
+            }
+        }
+    }
+    if (feature_map !== undefined && feature_map !== null) {
+        // A test file the ticket created (not one it edited) must be listed
+        // under a feature, so tests grow inside their feature's files (#583).
+        const created = changed_test_sources.filter(
+            ({ base, current: text }) => base === null && text !== null
+        )
+        if (created.length > 0 && feature_map.files === null) {
+            problems.push(
+                `The feature map ${feature_map.path} is missing: restore it, or add it with the new test file listed`
+            )
+        } else {
+            for (const { file } of created) {
+                if (!(feature_map.files ?? []).includes(file)) {
+                    problems.push(
+                        `${file}: a new test file the feature map doesn't list (add it under a feature in ${feature_map.path})`
+                    )
+                }
+            }
+        }
+    }
     const testFiles = new Set(test_files)
     for (const test of mapping.flatMap(({ tests }) => tests)) {
         const outcome = checkNewTest({
@@ -118,17 +295,8 @@ export const checkRed = ({
         if ('problem' in outcome) problems.push(outcome.problem)
         else notes.push(outcome.note)
     }
-    for (const old of baseline.cases.filter(
-        ({ status }) => status === 'passed'
-    )) {
-        const now = current.cases.find(
-            ({ file, full_name }) =>
-                file === old.file && full_name === old.full_name
-        )
-        const label = `old test "${old.full_name}" in ${old.file}`
-        if (now === undefined) problems.push(`${label} is missing`)
-        else if (now.status !== 'passed')
-            problems.push(`${label} no longer passes`)
-    }
+    const old = checkOldTests({ baseline, current })
+    problems.push(...old.problems)
+    notes.push(...old.notes)
     return { ok: problems.length === 0, problems, notes }
 }

package/engine/gates/shell-check.ts

@@ -0,0 +1,64 @@
+/**
+ * Runs one shell command as a named gate check (`shellCheck`), and the small
+ * clock the checks share (`msSince`). The install, prepare, types, lint, and
+ * `pass_fail` test checks all run through it.
+ */
+import type { GateCheck, GateName } from './gate-schemas'
+
+import {
+    clipOutput,
+    DEFAULT_COMMAND_TIMEOUT_MS,
+    runShell,
+    timedOutLine,
+} from '../shell/run-command'
+
+/** Milliseconds since `started` (from `performance.now()`), whole (#518). */
+export const msSince = (started: number): number =>
+    Math.round(performance.now() - started)
+
+/**
+ * Runs one shell command as a named check: `ok` on exit 0, with the clipped
+ * output only when it failed. One that runs past `timeout_ms` is stopped,
+ * with everything it started, and its output starts by saying so, then
+ * `timeout_hint`, if any. The check says how long the command ran (#518).
+ *
+ * @example
+ * const install = await shellCheck({ name: 'install', command: 'bun install', cwd })
+ */
+export const shellCheck = async ({
+    name,
+    command,
+    cwd,
+    timeout_ms,
+    timeout_hint,
+}: {
+    name: GateName
+    command: string
+    cwd: string
+    /** Defaults to `DEFAULT_COMMAND_TIMEOUT_MS`. */
+    timeout_ms?: number
+    /** How to give the command longer, for the timed-out message. */
+    timeout_hint?: string
+}): Promise<GateCheck> => {
+    const limit = timeout_ms ?? DEFAULT_COMMAND_TIMEOUT_MS
+    const started = performance.now()
+    const result = await runShell({ command, cwd, timeout_ms: limit })
+    const ok = result.exit_code === 0
+    const timedOut = result.timed_out
+        ? timedOutLine({
+              timeout_ms: limit,
+              hint: timeout_hint,
+              survivors: result.survivors,
+          })
+        : ''
+    return {
+        name,
+        command,
+        ok,
+        exit_code: result.exit_code,
+        output: ok
+            ? ''
+            : `${timedOut}${clipOutput({ text: `${result.stdout}\n${result.stderr}` })}`,
+        duration_ms: msSince(started),
+    }
+}

package/engine/gates/slot-waits.ts

@@ -0,0 +1,55 @@
+/**
+ * Waits inside a step (#632): a check that must wait for a prepare slot or
+ * a test slot says so to whoever listens for the step it runs in, such as
+ * the run's live file. The listener rides along with the step's work (an
+ * `AsyncLocalStorage`), so the checks deep in the gate runner need no new
+ * argument. Nothing is journaled: the wait's length is already `wait_ms`.
+ */
+import { AsyncLocalStorage } from 'node:async_hooks'
+
+/** What a check waits for. */
+export type SlotKind = 'prepare_slot' | 'test_slot'
+
+/** A wait for a slot, as it starts: `in_use` of the `slots` are taken. */
+export type SlotWait = { for: SlotKind; in_use: number; slots: number }
+
+/** Hears a wait start; the function it gives back hears the slot taken. */
+export type SlotWaitListener = (wait: SlotWait) => () => void
+
+const LISTENER = new AsyncLocalStorage<SlotWaitListener>()
+
+/**
+ * Runs `run` with `listener` hearing every slot wait inside it.
+ *
+ * @example
+ * await hearSlotWaits({ listener: live.slotWaits({ ticket: 11 }), run: () => runGates(...) })
+ */
+export const hearSlotWaits = <T>({
+    listener,
+    run,
+}: {
+    listener: SlotWaitListener
+    run: () => Promise<T>
+}): Promise<T> => LISTENER.run(listener, run)
+
+/**
+ * The `on_wait` for a take of `kind` slots, at most `limit` at once, or
+ * `undefined` when nobody listens. With no cap (`Infinity`), the slots are
+ * the ones in use.
+ */
+export const onSlotWait = ({
+    kind,
+    limit,
+}: {
+    kind: SlotKind
+    limit: number
+}): ((in_use: number) => () => void) | undefined => {
+    const listener = LISTENER.getStore()
+    if (listener === undefined) return undefined
+    return (in_use) =>
+        listener({
+            for: kind,
+            in_use,
+            slots: Number.isFinite(limit) ? limit : in_use,
+        })
+}

package/engine/gates/slow-test-runner.ts

@@ -0,0 +1,59 @@
+/**
+ * The final review's slow tests (#563): the prepare command, then each slow
+ * test command, as `slow_test` checks.
+ */
+import type { GateCheck } from './gate-schemas'
+import { prepareCheck } from './prepare-check'
+import { testChecks } from './test-checks'
+
+import {
+    slowTestCommands,
+    testTimeoutOf,
+    type EngineConfig,
+} from '../config/engine-config'
+
+/**
+ * The final review's slow tests (#563), once round 1's lenses are in: the
+ * prepare command, if any (a failed one stops there), then each slow
+ * test command, in order, as `slow_test` checks with `test_timeout_ms`.
+ * `around_prepare`, if given, runs the prepare check, as in `runGates`.
+ *
+ * @example
+ * const { ok, checks } = await runSlowTests({ cwd, config, test_files, report_file })
+ */
+export const runSlowTests = async ({
+    cwd,
+    config,
+    test_files,
+    report_file,
+    around_prepare,
+}: {
+    cwd: string
+    config: EngineConfig
+    test_files: string[]
+    report_file: string
+    /** Wraps the prepare run. Left out, it runs as is. */
+    around_prepare?: (
+        prepare: () => Promise<GateCheck | null>
+    ) => Promise<GateCheck | null>
+}): Promise<{ ok: boolean; checks: GateCheck[] }> => {
+    const prepare = () => prepareCheck({ config, cwd })
+    const prepared = await (around_prepare?.(prepare) ?? prepare())
+    if (prepared !== null && !prepared.ok) {
+        return { ok: false, checks: [prepared] }
+    }
+    const { checks } = await testChecks({
+        commands: slowTestCommands({ config }),
+        name: 'slow_test',
+        cwd,
+        test_files,
+        report_file,
+        first_index: 0,
+        timeout_ms: testTimeoutOf({ config }),
+        limit: null,
+        test_concurrency: config.test_concurrency,
+        exclusive: false,
+    })
+    const all = prepared === null ? checks : [prepared, ...checks]
+    return { ok: all.every(({ ok }) => ok), checks: all }
+}

package/engine/gates/test-checks.ts

@@ -0,0 +1,161 @@
+/**
+ * The test commands as gate checks (`testChecks`): `pass_fail` commands judged
+ * by their exit code, `bun` commands run with bun's JUnit reporter, each
+ * waiting for a test slot first. Also the fast test speed limit (#580).
+ */
+import type { GateCheck, TestRun } from './gate-schemas'
+import { msSince, shellCheck } from './shell-check'
+import {
+    reportFileFor,
+    runTestsInHeldSlot,
+    TEST_TIMEOUT_HINT,
+} from './test-runner'
+import { inTestSlot } from './test-slots'
+
+import type { TestCommand } from '../config/engine-config'
+
+/** What to do about a fast test over the speed limit (#580). */
+export const FAST_TEST_HINT = ({ limit_ms }: { limit_ms: number }): string =>
+    `A fast test may take at most ${limit_ms} ms (\`test_case_timeout_ms\`). Make it faster, or move it to the slow tests.`
+
+/** The fast test speed limit (#580), and the files measured against it. */
+export type SpeedLimit = { limit_ms: number; measured_files: string[] }
+
+/**
+ * A `bun` test command's run as a gate check named `name`: it fails on a
+ * failed run, or on a passing test over the fast test speed limit (see
+ * `testChecks`), and it ran from `started` until now.
+ */
+const bunCheck = ({
+    name,
+    run,
+    limit,
+    started,
+}: {
+    name: 'test' | 'slow_test'
+    run: TestRun
+    limit: SpeedLimit | null
+    started: number
+}): GateCheck => {
+    const tooSlow =
+        limit === null
+            ? []
+            : run.cases.filter(
+                  ({ file, status, duration_ms }) =>
+                      status === 'passed' &&
+                      limit.measured_files.includes(file) &&
+                      (duration_ms ?? 0) > limit.limit_ms
+              )
+    const timedOut = /timed out after \d+ms/.test(run.output)
+    const hint =
+        limit === null || (tooSlow.length === 0 && !timedOut)
+            ? ''
+            : `${FAST_TEST_HINT({ limit_ms: limit.limit_ms })}\n`
+    const slowLines = tooSlow
+        .map(
+            ({ file, full_name, duration_ms }) =>
+                `- ${file} > ${full_name}: ${duration_ms} ms`
+        )
+        .join('\n')
+    const ok = run.ok && tooSlow.length === 0
+    return {
+        name,
+        command: run.command,
+        ok,
+        exit_code: run.exit_code,
+        output: ok
+            ? ''
+            : `${hint}${slowLines === '' ? '' : `Too slow for a fast test:\n${slowLines}\n`}${run.output}`,
+        duration_ms: msSince(started),
+    }
+}
+
+/**
+ * Runs test commands as gate checks named `name`, in order (see `runGates`).
+ * With a `limit` (#580, the fast call only), each `bun` command gets
+ * `--timeout=<limit_ms>`, and a passing test in one of `measured_files` that
+ * took longer than that fails the check: a test's own timeout option
+ * overrides `--timeout`, so only its measured time tells.
+ *
+ * With `test_concurrency` (#544), each command first waits for a test slot
+ * (see `inTestSlot`); its check records the wait as `wait_ms`, and its time
+ * limit and `duration_ms` start after it. With `exclusive` (a lone rerun,
+ * #544), each command waits until no other test run is going, and holds
+ * every slot while it runs.
+ */
+export const testChecks = async ({
+    commands,
+    name,
+    cwd,
+    test_files,
+    report_file,
+    first_index,
+    timeout_ms,
+    limit,
+    test_concurrency,
+    exclusive,
+}: {
+    commands: TestCommand[]
+    name: 'test' | 'slow_test'
+    cwd: string
+    test_files: string[]
+    report_file: string
+    /** The first command's index, so each command gets its own report file. */
+    first_index: number
+    timeout_ms: number
+    /**
+     * The fast test speed limit (#580), or `null` for none. Only the fast
+     * call has one; slow tests never do.
+     */
+    limit: SpeedLimit | null
+    /** The config's `test_concurrency`; left out, no cap. */
+    test_concurrency: number | undefined
+    exclusive: boolean
+}): Promise<{ checks: GateCheck[]; runs: TestRun[] }> => {
+    const checks: GateCheck[] = []
+    const runs: TestRun[] = []
+    for (const [offset, { run: command, results }] of commands.entries()) {
+        if (results === 'pass_fail') {
+            checks.push(
+                await inTestSlot({
+                    test_concurrency,
+                    exclusive,
+                    work: () =>
+                        shellCheck({
+                            name,
+                            command,
+                            cwd,
+                            timeout_ms,
+                            timeout_hint: TEST_TIMEOUT_HINT,
+                        }),
+                })
+            )
+            continue
+        }
+        checks.push(
+            await inTestSlot({
+                test_concurrency,
+                exclusive,
+                work: async () => {
+                    const started = performance.now()
+                    const run = await runTestsInHeldSlot({
+                        cwd,
+                        command:
+                            limit === null
+                                ? command
+                                : `${command} --timeout=${limit.limit_ms}`,
+                        test_files,
+                        report_file: reportFileFor({
+                            report_file,
+                            index: first_index + offset,
+                        }),
+                        timeout_ms,
+                    })
+                    runs.push(run)
+                    return bunCheck({ name, run, limit, started })
+                },
+            })
+        )
+    }
+    return { checks, runs }
+}