@alecsibilia/luca 14.0.0-alpha.2 → 14.0.0-alpha.21

package/engine/gates/already-done-check.ts

@@ -0,0 +1,120 @@
+import uniqBy from 'lodash/uniqBy'
+
+import { endedText, type GateCheck, type TestRun } from './gate-schemas'
+import { findCase } from './red-check'
+
+import type { CriterionTests, TestRef } from '../agents/role-results'
+import type { CommitPlace } from '../git/git-adapter'
+
+/** What the engine's check of `already_done` evidence found (#495). */
+export type AlreadyDoneCheckResult = { ok: boolean; problems: string[] }
+
+/** What is wrong with one named test, or `null` if it passes on the base. */
+const testProblem = ({
+    test,
+    missing,
+    test_files,
+    changed,
+    run,
+}: {
+    test: TestRef
+    missing: Set<string>
+    test_files: Set<string>
+    changed: Set<string>
+    run: TestRun | null
+}): string | null => {
+    const label = `"${test.name}" in ${test.file}`
+    if (missing.has(test.file)) return `${label}: the file does not exist`
+    if (!test_files.has(test.file)) {
+        return `${label}: the file is not a test file`
+    }
+    if (changed.has(test.file)) {
+        return `${label}: the file has changes that are not on the base`
+    }
+    // No run: the prepare command failed, and says so itself.
+    if (run === null) return null
+    const found = findCase({ cases: run.cases, test })
+    if (found?.status === 'passed') return null
+    if (found?.status === 'skipped') return `${label}: is skipped`
+    if (found !== undefined) return `${label}: fails on the base`
+    return run.files_without_results.includes(test.file)
+        ? `${label}: its file did not load, so none of its tests ran`
+        : `${label}: not found in the test results`
+}
+
+/**
+ * The engine's check of a test-writer's `already_done` evidence (#495).
+ * Pure: every named commit is on the base (the commit the ticket's
+ * worktree started from), every criterion names a test, and every named
+ * test is in a test file that exists, is as it is on the base, and passes
+ * there.
+ *
+ * @param commits - Each named commit, and where git found it.
+ * @param missing - The named test files that do not exist in the worktree.
+ * @param test_files - The worktree's test files, per the engine config.
+ * @param changed - The worktree's uncommitted changes.
+ * @param prepare - The prepare command's run before the tests, if any.
+ * @param run - The run of the named test files; `null` if none ran.
+ *
+ * @example
+ * const result = checkAlreadyDone({ base_sha, criteria_ids: ['AC1'], criteria, commits, missing: [], test_files, changed: [], prepare: null, run })
+ * if (!result.ok) console.error(result.problems)
+ */
+export const checkAlreadyDone = ({
+    base_sha,
+    criteria_ids,
+    criteria,
+    commits,
+    missing,
+    test_files,
+    changed,
+    prepare,
+    run,
+}: {
+    base_sha: string
+    criteria_ids: string[]
+    criteria: CriterionTests[]
+    commits: { sha: string; place: CommitPlace }[]
+    missing: string[]
+    test_files: string[]
+    changed: string[]
+    prepare: GateCheck | null
+    run: TestRun | null
+}): AlreadyDoneCheckResult => {
+    const problems: string[] = []
+    for (const { sha, place } of commits) {
+        if (place === 'unknown') {
+            problems.push(`commit ${sha}: not found in the repo`)
+        } else if (place === 'not_on_base') {
+            problems.push(
+                `commit ${sha}: not on the base (${base_sha}, where this ticket started)`
+            )
+        }
+    }
+    for (const id of criteria_ids) {
+        const entry = criteria.find(({ criterion_id }) => criterion_id === id)
+        if (entry === undefined || entry.tests.length === 0) {
+            problems.push(`${id} names no test`)
+        }
+    }
+    if (prepare !== null && !prepare.ok) {
+        problems.push(
+            `The prepare command \`${prepare.command}\` failed (${endedText(prepare)}), so no test ran:\n${prepare.output}`
+        )
+    }
+    const tests = uniqBy(
+        criteria.flatMap(({ tests }) => tests),
+        ({ file, name }) => `${file}\0${name}`
+    )
+    for (const test of tests) {
+        const problem = testProblem({
+            test,
+            missing: new Set(missing),
+            test_files: new Set(test_files),
+            changed: new Set(changed),
+            run,
+        })
+        if (problem !== null) problems.push(problem)
+    }
+    return { ok: problems.length === 0, problems }
+}

package/engine/gates/feature-map.ts

@@ -0,0 +1,96 @@
+import { readdir } from 'node:fs/promises'
+import { join, normalize, relative } from 'node:path'
+
+import { testFilesAmong } from './test-runner'
+
+import { isFeatureMapFolder } from '../config/engine-config'
+import { pathText } from '../git/path-text'
+
+/**
+ * The test files a feature map lists (#583): every backticked path in it that
+ * matches the test file patterns, in order, once each. A path written with a
+ * leading `./` counts as the path without it, as a worktree-relative path is
+ * everywhere else.
+ *
+ * @param text - The feature map's markdown.
+ * @param test_file_patterns - The engine config's test file patterns.
+ * @returns The listed test files, in the order the map first names them.
+ *
+ * @example
+ * featureMapFiles({ text: '- `src/sum.test.ts`', test_file_patterns: ['**\/*.test.ts'] }) // ['src/sum.test.ts']
+ * featureMapFiles({ text: '- `./src/sum.test.ts`', test_file_patterns: ['**\/*.test.ts'] }) // ['src/sum.test.ts']
+ */
+export const featureMapFiles = ({
+    text,
+    test_file_patterns,
+}: {
+    text: string
+    test_file_patterns: string[]
+}): string[] => {
+    const paths = [...text.matchAll(/`([^`\s]+)`/g)].map(([, path = '']) =>
+        path.replace(/^\.\//, '')
+    )
+    return [...new Set(testFilesAmong({ files: paths, test_file_patterns }))]
+}
+
+/**
+ * The text of every `*.md` page under a folder, nested ones too, in sorted
+ * order of their paths (#597). `null` when the folder is missing or a page
+ * can't be read.
+ */
+const folderText = async ({
+    folder,
+}: {
+    folder: string
+}): Promise<string | null> => {
+    try {
+        const entries = await readdir(folder, {
+            recursive: true,
+            withFileTypes: true,
+        })
+        const pages = entries
+            .filter((entry) => entry.isFile() && entry.name.endsWith('.md'))
+            .map((entry) =>
+                relative(folder, join(entry.parentPath, entry.name))
+            )
+            .toSorted()
+        const texts = await Promise.all(
+            pages.map((page) => Bun.file(join(folder, page)).text())
+        )
+        return texts.join('\n')
+    } catch {
+        return null
+    }
+}
+
+/**
+ * The test files a worktree's feature map lists (#583, #597). A path ending
+ * in `.md` is one file, read as it always was; any other path is a folder,
+ * and every `*.md` page under it, in sorted order, together is the map.
+ * (`loadEngineConfig` refuses a map file whose name doesn't end in `.md`.)
+ *
+ * @param cwd - The worktree.
+ * @param path - The map's worktree-relative path, as the config spells it.
+ * @param test_file_patterns - The engine config's test file patterns.
+ * @returns The listed test files (see `featureMapFiles`), or `null` when the
+ *   map is missing or can't be read, so the red check can name it rather
+ *   than crash.
+ *
+ * @example
+ * await readFeatureMap({ cwd, path: 'docs/feature-map', test_file_patterns }) // ['src/sum.test.ts']
+ */
+export const readFeatureMap = async ({
+    cwd,
+    path,
+    test_file_patterns,
+}: {
+    cwd: string
+    path: string
+    test_file_patterns: string[]
+}): Promise<string[] | null> => {
+    const clean = normalize(path)
+    const text = isFeatureMapFolder({ path: clean })
+        ? await folderText({ folder: join(cwd, clean) })
+        : await pathText({ path: join(cwd, clean) })
+    return text === null ? null : featureMapFiles({ text, test_file_patterns })
+}

package/engine/gates/gate-runner.ts

@@ -1,100 +1,47 @@
-import type { GateCheck, GateName } from './gate-schemas'
-import { reportFileFor, runTests } from './test-runner'
+import type { GateCheck, GateName, TestCase } from './gate-schemas'
+import { ownFilesCheck } from './own-files-check'
+import { prepareCheck } from './prepare-check'
+import { shellCheck } from './shell-check'
+import { testChecks } from './test-checks'
 
 import {
-    prepareTimeoutOf,
+    caseLimitOf,
+    slowTestCommands,
     testCommands,
+    testTimeoutOf,
     type EngineConfig,
 } from '../config/engine-config'
-import {
-    clipOutput,
-    DEFAULT_COMMAND_TIMEOUT_MS,
-    runShell,
-} from '../shell/run-command'
-
-/** A time limit in words: `30 minutes`, `1 minute`, `0.3 seconds`. */
-const durationText = (ms: number): string => {
-    const [amount, unit] =
-        ms >= 60_000 ? [ms / 60_000, 'minute'] : [ms / 1000, 'second']
-    const rounded = Math.round(amount * 10) / 10
-    return `${rounded} ${unit}${rounded === 1 ? '' : 's'}`
-}
 
-/**
- * Runs one shell command as a named check: `ok` on exit 0, with the clipped
- * output only when it failed. One that runs past `timeout_ms` is stopped,
- * with everything it started, and its output starts by saying so, then
- * `timeout_hint`, if any.
- *
- * @example
- * const install = await shellCheck({ name: 'install', command: 'bun install', cwd })
- */
-export const shellCheck = async ({
-    name,
-    command,
-    cwd,
-    timeout_ms,
-    timeout_hint,
-}: {
-    name: GateName
-    command: string
-    cwd: string
-    /** Defaults to `DEFAULT_COMMAND_TIMEOUT_MS`. */
-    timeout_ms?: number
-    /** How to give the command longer, for the timed-out message. */
-    timeout_hint?: string
-}): Promise<GateCheck> => {
-    const limit = timeout_ms ?? DEFAULT_COMMAND_TIMEOUT_MS
-    const result = await runShell({ command, cwd, timeout_ms: limit })
-    const ok = result.exit_code === 0
-    const timedOut = result.timed_out
-        ? `Timed out after ${durationText(limit)}, so the engine stopped it and everything it started.${timeout_hint === undefined ? '' : ` ${timeout_hint}`}\n`
-        : ''
-    return {
-        name,
-        command,
-        ok,
-        exit_code: result.exit_code,
-        output: ok
-            ? ''
-            : `${timedOut}${clipOutput({ text: `${result.stdout}\n${result.stderr}` })}`,
-    }
-}
-
-/**
- * Runs the config's prepare command in `cwd`, as the `prepare` check, or
- * `null` when the config has none. The engine runs it before every test run
- * in a checkout, so tests that need build outputs find them up to date.
- * It gets its own time limit (see `prepareTimeoutOf`), since a first build
- * can take far longer than other commands.
- *
- * @example
- * const prepare = await prepareCheck({ config, cwd })
- * if (prepare !== null && !prepare.ok) console.log(prepare.output)
- */
-export const prepareCheck = async ({
-    config,
-    cwd,
-}: {
-    config: EngineConfig
-    cwd: string
-}): Promise<GateCheck | null> =>
-    config.prepare === undefined
-        ? null
-        : shellCheck({
-              name: 'prepare',
-              command: config.prepare,
-              cwd,
-              timeout_ms: prepareTimeoutOf({ config }),
-              timeout_hint:
-                  'To give it longer, raise `prepare_timeout_ms` in `.luca/config.json`.',
-          })
+export { prepareCheck } from './prepare-check'
+export { runSlowTests } from './slow-test-runner'
+export { shellCheck } from './shell-check'
+export { FAST_TEST_HINT } from './test-checks'
 
 /**
  * Runs every gate the engine config names, in order: each test command,
- * types, lint. All must pass. Unset gates are left out; intake refuses a
+ * then (the final review's gates only) each slow test command, types, lint.
+ * All must pass. Unset gates are left out; intake refuses a
  * config with no test command. A `bun` test command runs with bun's JUnit
  * reporter; a `pass_fail` one runs as written and is judged by its exit code.
+ * Each test command gets the config's time limit (see `testTimeoutOf`);
+ * types and lint get the engine's default. With `test_concurrency` (#544),
+ * each test command (the own-files run too) waits for a test slot first,
+ * and its check records the wait as `wait_ms`.
+ *
+ * With a fast test speed limit (`test_case_timeout_ms`, #580), every `bun`
+ * command of `checks.test` gets `--timeout=<limit>`, and the tests in
+ * `measured_files` are also held to it by measured time. Slow commands, the
+ * own-files run, and `pass_fail` commands never get it.
+ *
+ * With `own_test_files` (a ticket's gates, with a slow command, #564), the
+ * ones the fast runs gave no results for run next on their own with plain
+ * `bun test`, as one more `test` check, with the same time limit. A run that
+ * gives no results (its files don't load) adds no check after a failed fast
+ * run, which already fails the gate.
+ *
+ * Also returns `cases`, the test cases of the fast `bun` runs and the
+ * own-files run, each with its `expect()` count, so the engine can find a
+ * mapped test that checks nothing (#581). A `pass_fail` command has none.
  *
  * With an `install` command (see `installCommand`), the engine runs it first,
  * as the `install` check. A failed install stops there: the other gates
@@ -103,8 +50,11 @@ export const prepareCheck = async ({
  * `around_prepare`, if given, runs the prepare check (so the engine can see
  * what it made).
  *
+ * With `exclusive` (a lone rerun, #544), each test command waits until no
+ * other test run is going, and holds every test slot while it runs.
+ *
  * @example
- * const { ok, checks } = await runGates({ cwd, config, test_files, report_file, install: null })
+ * const { ok, checks, cases } = await runGates({ cwd, config, test_files, report_file, install: null })
  */
 export const runGates = async ({
     cwd,
@@ -113,6 +63,10 @@ export const runGates = async ({
     report_file,
     install,
     around_prepare,
+    with_slow_tests,
+    own_test_files,
+    measured_files,
+    exclusive = false,
 }: {
     cwd: string
     config: EngineConfig
@@ -123,7 +77,24 @@ export const runGates = async ({
     around_prepare?: (
         prepare: () => Promise<GateCheck | null>
     ) => Promise<GateCheck | null>
-}): Promise<{ ok: boolean; checks: GateCheck[] }> => {
+    /**
+     * The final review's gates also run the config's slow tests (#563), after
+     * the test commands, as `slow_test` checks. Left out, they don't.
+     */
+    with_slow_tests?: boolean
+    /**
+     * A ticket's changed test files (#564): those no fast run gave results
+     * for are run on their own, as one more `test` check. Left out, none.
+     */
+    own_test_files?: string[]
+    /**
+     * A ticket's changed test files: their tests are held to
+     * `test_case_timeout_ms` by measured time (#580); left out, none.
+     */
+    measured_files?: string[]
+    /** Run each test command alone (#544). Left out, they don't. */
+    exclusive?: boolean
+}): Promise<{ ok: boolean; checks: GateCheck[]; cases: TestCase[] }> => {
     const checks: GateCheck[] = []
     if (install !== null) {
         const installed = await shellCheck({
@@ -132,35 +103,64 @@ export const runGates = async ({
             cwd,
         })
         checks.push(installed)
-        if (!installed.ok) return { ok: false, checks }
+        if (!installed.ok) return { ok: false, checks, cases: [] }
     }
     const prepare = () => prepareCheck({ config, cwd })
     const prepared = await (around_prepare?.(prepare) ?? prepare())
     if (prepared !== null) {
         checks.push(prepared)
-        if (!prepared.ok) return { ok: false, checks }
+        if (!prepared.ok) return { ok: false, checks, cases: [] }
     }
     const { types, lint } = config.checks
-    for (const [index, { run: command, results }] of testCommands({
-        config,
-    }).entries()) {
-        if (results === 'pass_fail') {
-            checks.push(await shellCheck({ name: 'test', command, cwd }))
-            continue
-        }
-        const run = await runTests({
+    const timeout_ms = testTimeoutOf({ config })
+    const { test_concurrency } = config
+    const limit_ms = caseLimitOf({ config })
+    const fast = testCommands({ config })
+    const fastChecks = await testChecks({
+        commands: fast,
+        name: 'test',
+        cwd,
+        test_files,
+        report_file,
+        first_index: 0,
+        timeout_ms,
+        limit:
+            limit_ms === null
+                ? null
+                : { limit_ms, measured_files: measured_files ?? [] },
+        test_concurrency,
+        exclusive,
+    })
+    checks.push(...fastChecks.checks)
+    // The test cases the gates saw, for the empty-test check (#581).
+    const cases = fastChecks.runs.flatMap((run) => run.cases)
+    const own = await ownFilesCheck({
+        own_test_files: own_test_files ?? [],
+        fast_checks: fastChecks.checks,
+        fast_runs: fastChecks.runs,
+        cwd,
+        report_file,
+        report_index: fast.length + slowTestCommands({ config }).length,
+        timeout_ms,
+        test_concurrency,
+        exclusive,
+    })
+    cases.push(...own.cases)
+    if (own.check !== null) checks.push(own.check)
+    if (with_slow_tests === true) {
+        const slow = await testChecks({
+            commands: slowTestCommands({ config }),
+            name: 'slow_test',
             cwd,
-            command,
             test_files,
-            report_file: reportFileFor({ report_file, index }),
-        })
-        checks.push({
-            name: 'test',
-            command,
-            ok: run.ok,
-            exit_code: run.exit_code,
-            output: run.ok ? '' : run.output,
+            report_file,
+            first_index: fast.length,
+            timeout_ms,
+            limit: null,
+            test_concurrency,
+            exclusive,
         })
+        checks.push(...slow.checks)
     }
     const others: { name: GateName; command: string | undefined }[] = [
         { name: 'types', command: types },
@@ -170,5 +170,5 @@ export const runGates = async ({
         if (command === undefined) continue
         checks.push(await shellCheck({ name, command, cwd }))
     }
-    return { ok: checks.every(({ ok }) => ok), checks }
+    return { ok: checks.every(({ ok }) => ok), checks, cases }
 }

package/engine/gates/gate-schemas.ts

@@ -6,10 +6,22 @@ export const TestCaseSchema = z.object({
     /** Describe names and the test name joined by " > ". */
     full_name: z.string(),
     status: z.enum(['passed', 'failed', 'skipped']),
+    /** How long it ran, from bun's JUnit `time` (#518). Missing in old journals. */
+    duration_ms: z.number().int().nonnegative().optional(),
+    /** How many `expect()` calls it made, from bun's JUnit `assertions` (#581). */
+    assertions: z.number().int().nonnegative().optional(),
 })
 
 export type TestCase = z.infer<typeof TestCaseSchema>
 
+/** A test case by its file and full name, without its outcome. */
+export const TestCaseRefSchema = TestCaseSchema.pick({
+    file: true,
+    full_name: true,
+})
+
+export type TestCaseRef = z.infer<typeof TestCaseRefSchema>
+
 /** One run of the engine config's test command, with each case's outcome. */
 export const TestRunSchema = z.object({
     command: z.string(),
@@ -22,21 +34,31 @@ export const TestRunSchema = z.object({
     files_without_results: z.array(z.string()),
     /** The command's output, clipped. */
     output: z.string(),
+    /**
+     * How long it waited for a test slot (#544), only with
+     * `test_concurrency` set.
+     */
+    wait_ms: z.number().int().nonnegative().optional(),
 })
 
 export type TestRun = z.infer<typeof TestRunSchema>
 
 /**
  * The gates the engine config can name, plus `install`: the engine's own
- * install, run first when a package manifest changed, and `prepare`: the
- * config's prepare command, run before the tests.
+ * install, run first when a package manifest changed, `prepare`: the
+ * config's prepare command, run before the tests, and `slow_test`: the
+ * config's slow tests, run only at the final review (#563). `generate` is
+ * one of the config's generators, run after a writer's turn when it asked
+ * (#618); it is journaled in `chores_done`, never in the gates.
  */
 export const GateNameSchema = z.enum([
     'install',
     'prepare',
     'test',
+    'slow_test',
     'types',
     'lint',
+    'generate',
 ])
 
 export type GateName = z.infer<typeof GateNameSchema>
@@ -48,6 +70,14 @@ export const GateCheckSchema = z.object({
     ok: z.boolean(),
     exit_code: z.number().int().nullable(),
     output: z.string(),
+    /** How long the command ran (#518). Missing in old journals. */
+    duration_ms: z.number().int().nonnegative().optional(),
+    /**
+     * How long it waited for its slot: a prepare slot for `prepare` (#518),
+     * a test slot for `test` and `slow_test` with `test_concurrency` set
+     * (#544).
+     */
+    wait_ms: z.number().int().nonnegative().optional(),
 })
 
 export type GateCheck = z.infer<typeof GateCheckSchema>
@@ -82,3 +112,32 @@ export const LeftoverHitSchema = z.object({
 })
 
 export type LeftoverHit = z.infer<typeof LeftoverHitSchema>
+
+/**
+ * Why the leftover scan let a new markdown file through (#520): it sits in a
+ * doc place (a `docs` or `adr` folder, or a `README.md`), a tracked or newly
+ * added non-test file names it, or a `leftover_allow` pattern matches it.
+ */
+export const LeftoverPassReasonSchema = z.enum([
+    'doc_place',
+    'named_by_repo',
+    'config_pattern',
+])
+
+export type LeftoverPassReason = z.infer<typeof LeftoverPassReasonSchema>
+
+/**
+ * A new markdown file the leftover scan let through, and why, as the journal
+ * keeps it. The reason is a string there, so a later reason still reads.
+ */
+export const LeftoverPassSchema = z.object({
+    path: z.string(),
+    reason: z.string(),
+})
+
+/** A new markdown file the leftover scan let through, as the scan returns it. */
+export const LeftoverPassResultSchema = LeftoverPassSchema.extend({
+    reason: LeftoverPassReasonSchema,
+})
+
+export type LeftoverPass = z.infer<typeof LeftoverPassResultSchema>