@alecsibilia/luca 14.0.0-alpha.32 → 14.0.0-alpha.34

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @alecsibilia/luca
 
+## 14.0.0-alpha.34
+
+No changes in this release.
+
+## 14.0.0-alpha.33
+
+No changes in this release.
+
 ## 14.0.0-alpha.32
 
 No changes in this release.

package/board/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@luca/board",
-    "version": "14.0.0-alpha.32",
+    "version": "14.0.0-alpha.34",
     "private": true,
     "type": "module"
 }

package/board/server/board-vocabulary-engine.ts

@@ -35,7 +35,15 @@ export const ENGINE_VOCABULARY = {
     gates_run: z.looseObject({
         target: z.string(),
         ok: z.boolean(),
-        checks: z.array(z.looseObject({ name: z.string(), ok: z.boolean() })),
+        /** `skipped`: a ticket gate didn't run the test command (#722). */
+        checks: z.array(
+            z.looseObject({
+                name: z.string(),
+                ok: z.boolean(),
+                command: z.string().optional(),
+                skipped: z.boolean().optional(),
+            })
+        ),
     }),
     ticket_joined: z.looseObject({
         ok: z.boolean(),

package/board/server/describe-record.ts

@@ -238,13 +238,21 @@ export const describeRecord = ({
                       ? ' after joining'
                       : ''
             const who = record.ticket === null ? 'Final review: ' : at
+            // The test commands a ticket gate skipped by path (#722).
+            const skipped = record.content.checks
+                .filter((check) => check.skipped === true)
+                .map((check) => check.command ?? check.name)
+            const skips =
+                record.content.target === 'ticket' && skipped.length > 0
+                    ? ` Skipped (no changed file matches its paths): ${skipped.join(', ')}.`
+                    : ''
             return record.content.ok
                 ? event({
-                      text: `${who}checks passed${where}.`,
+                      text: `${who}checks passed${where}.${skips}`,
                       tone: 'success',
                   })
                 : event({
-                      text: `${who}checks failed${where}: ${failed.join(', ') || 'no detail'}.`,
+                      text: `${who}checks failed${where}: ${failed.join(', ') || 'no detail'}.${skips}`,
                       tone: 'danger',
                   })
         }

package/board/server/luca-version.ts

@@ -1,2 +1,2 @@
 /** The Luca version of this board, stamped when the package was packed. */
-export const LUCA_VERSION: string = "14.0.0-alpha.32"
+export const LUCA_VERSION: string = "14.0.0-alpha.34"

package/engine/cli/setup-build-cache-check.ts

@@ -0,0 +1,52 @@
+/**
+ * Doctor's `build_cache` check (#721): each `build_cache_paths` path should
+ * be ignored by git in the repo's checkout, since Luca never copies one
+ * that isn't, so a copied cache never ends up in a commit.
+ */
+import uniq from 'lodash/uniq'
+
+import { ok, warning, type Found } from './doctor-checks'
+
+import { buildCachePathsOf, type EngineConfig } from '../config/engine-config'
+import { ignoredAmong, isGlob, matchesIn } from '../core/build-cache'
+import { createGitAdapter } from '../git/git-adapter'
+
+/**
+ * The `build_cache` check for a config with build cache paths: a warning
+ * naming each path git doesn't ignore in the checkout, made into one; else
+ * OK. A plain entry is checked as written, even before its folder is there;
+ * a glob, by its matches in the checkout.
+ */
+export const checkBuildCache = async ({
+    repo,
+    config,
+}: {
+    repo: string
+    config: EngineConfig
+}): Promise<Found> => {
+    const paths: string[] = []
+    for (const entry of buildCachePathsOf({ config })) {
+        paths.push(
+            ...(isGlob(entry) ? await matchesIn({ entry, dir: repo }) : [entry])
+        )
+    }
+    const checked = uniq(paths)
+    const ignored = await ignoredAmong({
+        git: createGitAdapter({ repo_root: repo }),
+        cwd: repo,
+        paths: checked,
+    })
+    const missing = checked.filter((path) => !ignored.has(path))
+    if (missing.length === 0) {
+        return ok('Each build_cache_paths path is ignored by git')
+    }
+    return warning({
+        detail: missing
+            .map(
+                (path) =>
+                    `\`build_cache_paths\` path ${path} isn't ignored by git, so Luca won't copy it.`
+            )
+            .join(' '),
+        fix: missing.map((path) => `Add ${path} to .gitignore.`).join(' '),
+    })
+}

package/engine/cli/setup-repo-checks.ts

@@ -1,6 +1,6 @@
 /**
  * Doctor's repo group (`repoChecks`), which `luca setup` runs once too: the
- * labels, the config, its junit commands (#668), the GitHub remote, issue
+ * labels, the config, its junit commands (#668) and build caches (#721), the GitHub remote, issue
  * links, the base branch on `origin`, and the memory vault. Read-only.
  */
 import { join } from 'node:path'
@@ -12,10 +12,12 @@ import {
     type DoctorCheck,
     type Found,
 } from './doctor-checks'
+import { checkBuildCache } from './setup-build-cache-check'
 import { errorText, findConfig } from './setup-config'
 import { checkJunit } from './setup-junit-check'
 
 import {
+    buildCachePathsOf,
     ENGINE_CONFIG_FILE,
     hasJunitCommand,
     loadEngineConfig,
@@ -223,7 +225,8 @@ const checkConfig = async ({
 /**
  * `luca setup`'s checks as doctor's repo group, read-only: they create no
  * label and write no file. Each problem carries its fix. The `junit` check
- * (#668) is there only when the config loads and has a junit command.
+ * (#668) is there only when the config loads and has a junit command, and
+ * the `build_cache` check (#721) only when it has build cache paths.
  *
  * @example
  * const checks = await repoChecks({ repo: process.cwd(), github, memory })
@@ -248,12 +251,22 @@ export const repoChecks = async ({
         loaded.ok && hasJunitCommand({ config: loaded.config })
             ? [['junit', () => checkJunit({ repo, config: loaded.config })]]
             : []
+    const build_cache: [string, () => Promise<Found>][] =
+        loaded.ok && buildCachePathsOf({ config: loaded.config }).length > 0
+            ? [
+                  [
+                      'build_cache',
+                      () => checkBuildCache({ repo, config: loaded.config }),
+                  ],
+              ]
+            : []
     return checksFor({
         group: 'repo',
         checks: [
             ['labels', () => checkLabels({ repo, github, github_repo })],
             ['config', () => checkConfig({ repo, loaded })],
             ...junit,
+            ...build_cache,
             ['github_remote', async () => checkGitHubRemote({ github_repo })],
             ['issue_links', () => checkIssueLinks({ github, github_repo })],
             ['base_branch', () => checkBaseBranch({ repo, base_branch })],

package/engine/config/config-fields.ts

@@ -104,6 +104,19 @@ export const modelsOf = ({ config }: { config: EngineConfig }): ModelsConfig =>
 export const autoRetryOf = ({ config }: { config: EngineConfig }): boolean =>
     config.auto_retry ?? true
 
+/**
+ * The repo's build-cache paths and globs (#721): the config's
+ * `build_cache_paths`, else none.
+ *
+ * @example
+ * buildCachePathsOf({ config: { ...config, build_cache_paths: ['cache'] } }) // ['cache']
+ */
+export const buildCachePathsOf = ({
+    config,
+}: {
+    config: EngineConfig
+}): string[] => config.build_cache_paths ?? []
+
 /**
  * The feature map's path (#583), in the form a worktree-relative path takes
  * everywhere else (`normalize`, no leading `./`), or `null` with no map.

package/engine/config/config-reload.ts

@@ -3,7 +3,7 @@ import omit from 'lodash/omit'
 import reduce from 'lodash/reduce'
 import { z } from 'zod'
 
-import { autoRetryOf } from './config-fields'
+import { autoRetryOf, buildCachePathsOf } from './config-fields'
 import type { EngineConfig } from './engine-config'
 
 /**
@@ -32,6 +32,9 @@ import type { EngineConfig } from './engine-config'
  * - `auto_retry`: whether the engine retries a routine stuck item once by
  *   itself (#705). Only an item stuck after the resume is retried, so
  *   turning it on leaves an item already told to the owner waiting.
+ * - `build_cache_paths`: the build caches cloned into each new ticket
+ *   worktree (#721). Each new worktree reads it then, so a new value counts
+ *   from the next ticket started.
  */
 export const RELOADED_FIELDS = [
     'prepare',
@@ -42,6 +45,7 @@ export const RELOADED_FIELDS = [
     'test_case_timeout_ms',
     'leftover_allow',
     'auto_retry',
+    'build_cache_paths',
 ] as const
 
 /**
@@ -141,6 +145,12 @@ export const ConfigChangeSchema = z.discriminatedUnion('field', [
         from: z.boolean(),
         to: z.boolean(),
     }),
+    // Never left out: a config without it reads as `[]`.
+    z.object({
+        field: z.literal('build_cache_paths'),
+        from: z.array(z.string().min(1)),
+        to: z.array(z.string().min(1)),
+    }),
 ])
 
 export type ConfigChange = z.infer<typeof ConfigChangeSchema>
@@ -202,6 +212,11 @@ export const configChanges = ({
             from: autoRetryOf({ config: current }),
             to: autoRetryOf({ config: next }),
         },
+        {
+            field: 'build_cache_paths',
+            from: buildCachePathsOf({ config: current }),
+            to: buildCachePathsOf({ config: next }),
+        },
     ]
     return all.filter(({ from, to }) => !isEqual(from, to))
 }
@@ -244,6 +259,8 @@ const applyChange = ({
                 : { ...config, leftover_allow: change.to }
         case 'auto_retry':
             return { ...config, auto_retry: change.to }
+        case 'build_cache_paths':
+            return { ...config, build_cache_paths: change.to }
     }
 }
 

package/engine/config/engine-config-schema.ts

@@ -45,12 +45,64 @@ const isFileInsideRepo = (path: string): boolean => {
     )
 }
 
+/**
+ * Whether a `build_cache_paths` entry names a path or glob inside the repo
+ * (#721): not empty, not absolute, and, once cleaned, neither the repo root
+ * nor a path that climbs out of it with `..`.
+ */
+const isCachePathInsideRepo = (path: string): boolean => {
+    const clean = normalize(path)
+    return (
+        path !== '' &&
+        !isAbsolute(path) &&
+        clean !== '.' &&
+        clean !== './' &&
+        clean !== '..' &&
+        !clean.startsWith('../')
+    )
+}
+
+/** One `build_cache_paths` entry, its trailing `/` dropped. */
+const BuildCachePathSchema = z
+    .string()
+    .refine(isCachePathInsideRepo, {
+        message:
+            'each build_cache_paths entry must be a path or glob inside the repo, relative to its root: not empty, not an absolute path, not the repo root, and not one that leaves the repo with ".."',
+    })
+    .transform((path) => path.replace(/\/+$/, ''))
+
 /** One JUnit file a junit test command writes, relative to the repo root. */
 const JunitPathSchema = z.string().min(1).refine(isFileInsideRepo, {
     message:
         'must be a file inside the repo, relative to its root: not an absolute path, not one that leaves the repo with "..", and not ending in "/"',
 })
 
+/**
+ * Whether a `paths` glob stays inside the repo (#722): not absolute, and,
+ * once cleaned (`normalize`), not one that climbs out of it with `..`.
+ */
+const isGlobInsideRepo = (glob: string): boolean => {
+    const clean = normalize(glob)
+    return !isAbsolute(glob) && clean !== '..' && !clean.startsWith('../')
+}
+
+/**
+ * The paths a `checks.test` command covers (#722): repo-relative globs. At a
+ * ticket's gates the command runs only when the ticket changed a file under
+ * them.
+ */
+const TestPathsSchema = z
+    .array(
+        z.string().refine((glob) => glob !== '' && isGlobInsideRepo(glob), {
+            message:
+                'each paths entry must be a glob inside the repo, relative to its root: not empty, not an absolute path, and not one that leaves the repo with ".."',
+        })
+    )
+    .min(1, {
+        message:
+            'paths must list at least one glob; leave paths out to run the command at every gate',
+    })
+
 /** The text a slow command's `run_files` must hold, where the files go (#683). */
 export const RUN_FILES_PLACEHOLDER = '{files}'
 
@@ -60,7 +112,8 @@ export const RUN_FILES_PLACEHOLDER = '{files}'
  * `junit_path` (#668), one path or a list; no other kind may have one. A slow
  * entry may also name the test files it owns (`file_patterns`) and how to run
  * some of them (`run_files`, #683); they come together, and a `checks.test`
- * entry may have neither.
+ * entry may have neither. Only a `checks.test` entry may name the paths it
+ * covers (`paths`, #722).
  */
 const testCommandEntrySchema = ({ slow }: { slow: boolean }) =>
     z.union([
@@ -74,6 +127,11 @@ const testCommandEntrySchema = ({ slow }: { slow: boolean }) =>
                     .optional(),
                 file_patterns: z.array(z.string().min(1)).min(1).optional(),
                 run_files: z.string().min(1).optional(),
+                paths: TestPathsSchema.optional(),
+            })
+            .refine(({ paths }) => !slow || paths === undefined, {
+                message:
+                    'checks.slow_test: paths belongs only on checks.test entries.',
             })
             .refine(
                 ({ file_patterns, run_files }) =>
@@ -351,6 +409,14 @@ export const EngineConfigSchema = z.object({
      * as `true` too (`autoRetryOf`).
      */
     auto_retry: z.boolean().default(true).optional(),
+    /**
+     * The repo's build-cache folders (#721), as paths or globs relative to
+     * the repo root, such as `packages/ios/.derived`: each new ticket
+     * worktree gets the ones git ignores cloned in from the run branch's
+     * worktree, or else the repo's checkout. Left out, `[]`; a config
+     * written before it may lack it (`buildCachePathsOf`).
+     */
+    build_cache_paths: z.array(BuildCachePathSchema).default([]).optional(),
 })
 
 export type EngineConfig = z.infer<typeof EngineConfigSchema>

package/engine/config/engine-config.ts

@@ -18,6 +18,7 @@ import {
 
 export {
     autoRetryOf,
+    buildCachePathsOf,
     DEFAULT_PREPARE_CONCURRENCY,
     DEFAULT_PREPARE_TIMEOUT_MS,
     decisionModelOf,
@@ -60,6 +61,7 @@ export {
     ownSlowCommands,
     slowTestCommands,
     testCommands,
+    testCommandsToRun,
     type OwnSlowCommand,
     type TestCommand,
 } from './test-commands'

package/engine/config/test-commands.ts

@@ -8,12 +8,14 @@ import type { EngineConfig, TestResults } from './engine-config-schema'
 /**
  * One test command the engine runs, how it reads the results, and the JUnit
  * files a `junit` command writes (#668), relative to the repo root: `[]`
- * for every other kind.
+ * for every other kind. `paths` are the globs it covers at a ticket's gates
+ * (#722), or `null` when it covers everything.
  */
 export type TestCommand = {
     run: string
     results: TestResults
     junit_path: string[]
+    paths: string[] | null
 }
 
 /** Whether `run` is a `bun test` command, which gives bun's per-test results. */
@@ -36,7 +38,12 @@ const commandsOf = (value: EngineConfig['checks']['test']): TestCommand[] => {
     if (value === undefined) return []
     return (typeof value === 'string' ? [value] : value).map((entry) =>
         typeof entry === 'string'
-            ? { run: entry, results: defaultResults(entry), junit_path: [] }
+            ? {
+                  run: entry,
+                  results: defaultResults(entry),
+                  junit_path: [],
+                  paths: null,
+              }
             : {
                   run: entry.run,
                   results: entry.results ?? defaultResults(entry.run),
@@ -44,10 +51,56 @@ const commandsOf = (value: EngineConfig['checks']['test']): TestCommand[] => {
                       typeof entry.junit_path === 'string'
                           ? [entry.junit_path]
                           : (entry.junit_path ?? []),
+                  paths: entry.paths ?? null,
               }
     )
 }
 
+/** Whether `file` matches one of the globs. */
+const matchesAny = ({
+    file,
+    globs,
+}: {
+    file: string
+    globs: string[]
+}): boolean => globs.some((glob) => new Bun.Glob(glob).match(file))
+
+/**
+ * The test commands a ticket's gate runs, and the ones it skips (#722), each
+ * in config order. A command with no `paths` always runs; one with `paths`
+ * runs only when a changed file matches one of them. If a changed test file
+ * (by `test_file_patterns`) matches no command's `paths`, every command
+ * runs, so a ticket's own tests always run at its gates.
+ *
+ * @example
+ * testCommandsToRun({ commands: [a, b], changed_files: ['src/b/x.ts'], test_file_patterns: ['**\/*.test.ts'] })
+ * // { run: [b], skipped: [a] } when a has paths ['src/a/**'] and b ['src/b/**']
+ */
+export const testCommandsToRun = ({
+    commands,
+    changed_files,
+    test_file_patterns,
+}: {
+    commands: TestCommand[]
+    changed_files: string[]
+    test_file_patterns: string[]
+}): { run: TestCommand[]; skipped: TestCommand[] } => {
+    const covered = commands.flatMap(({ paths }) => paths ?? [])
+    const uncoveredTest = changed_files.some(
+        (file) =>
+            matchesAny({ file, globs: test_file_patterns }) &&
+            !matchesAny({ file, globs: covered })
+    )
+    if (uncoveredTest) return { run: commands, skipped: [] }
+    const runs = ({ paths }: TestCommand): boolean =>
+        paths === null ||
+        changed_files.some((file) => matchesAny({ file, globs: paths }))
+    return {
+        run: commands.filter(runs),
+        skipped: commands.filter((command) => !runs(command)),
+    }
+}
+
 /**
  * Every test command in the config, in order, each with its results kind
  * and `junit_path`. A missing `results` is `bun` for a `bun test` command,

package/engine/core/build-cache-copy.ts

@@ -0,0 +1,206 @@
+/**
+ * Warm builds (#721): `create_ticket_worktree` clones the repo's
+ * `build_cache_paths` into a ticket's new worktree, from the run branch's
+ * worktree or else the repo's checkout, both only read. Each copy goes to a
+ * temporary folder in the run folder first and is then renamed into place,
+ * so a worktree never holds a half-made copy. Nothing here ever stops the
+ * ticket: a failed scan, git check, copy or temporary folder only skips
+ * paths, each journaled with its reason.
+ */
+import { existsSync } from 'node:fs'
+import { mkdir, rename, rm } from 'node:fs/promises'
+import { dirname, join } from 'node:path'
+
+import {
+    copyDir,
+    ignoredAmong,
+    isGlob,
+    matchesIn,
+    type CopyDirResult,
+} from './build-cache'
+import type { BuildContext } from './build-context'
+
+import { buildCachePathsOf } from '../config/engine-config'
+import type {
+    BuildCacheSkipReason,
+    BuildCacheSource,
+} from '../journal/build-cache-entries'
+
+/** One source folder a cache may be copied from. */
+type Source = { from: BuildCacheSource; dir: string }
+
+/** One match of an entry, in the first source that has it. */
+type Match = Source & { path: string }
+
+/** An error's text. */
+const messageOf = (error: unknown): string =>
+    error instanceof Error ? error.message : String(error)
+
+/** Removes a folder; one it can't remove is left, and never throws. */
+const removeQuietly = (dir: string): Promise<void> =>
+    rm(dir, { recursive: true, force: true }).catch(() => undefined)
+
+/** The run's two warm sources, in order: the run branch's worktree, then the checkout. */
+const sourcesOf = ({ context }: { context: BuildContext }): Source[] => {
+    const started = context.journal
+        .read()
+        .find((record) => record.kind === 'run_started')
+    const repo = started?.kind === 'run_started' ? started.content.repo : null
+    const run_branch = context.state.run_branch?.path ?? null
+    const sources: Source[] = []
+    if (run_branch !== null) {
+        sources.push({ from: 'run_branch', dir: run_branch })
+    }
+    if (repo !== null) sources.push({ from: 'checkout', dir: repo })
+    return sources
+}
+
+/**
+ * An entry's matches, each from the first source that has it. A source it
+ * can't scan has no match.
+ */
+const matchesOf = async ({
+    entry,
+    sources,
+}: {
+    entry: string
+    sources: Source[]
+}): Promise<Match[]> => {
+    const found: Match[] = []
+    for (const source of sources) {
+        const paths = await matchesIn({ entry, dir: source.dir }).catch(
+            (): string[] => []
+        )
+        for (const path of paths) {
+            if (!found.some((match) => match.path === path)) {
+                found.push({ ...source, path })
+            }
+            if (!isGlob(entry)) return found
+        }
+    }
+    return found
+}
+
+/** Copies a match to `temp`, then renames it into the worktree. */
+const copyInto = async ({
+    context,
+    match,
+    temp,
+    worktree,
+}: {
+    context: BuildContext
+    match: Match
+    temp: string
+    worktree: string
+}): Promise<CopyDirResult> => {
+    const copy = context.copy_dir ?? copyDir
+    try {
+        const copied = await copy({
+            from: join(match.dir, match.path),
+            to: temp,
+        })
+        if (!copied.ok) return copied
+        const target = join(worktree, match.path)
+        await mkdir(dirname(target), { recursive: true })
+        await rename(temp, target)
+        return { ok: true }
+    } catch (error) {
+        return { ok: false, error: messageOf(error) }
+    } finally {
+        await removeQuietly(temp)
+    }
+}
+
+/**
+ * Clones the config's build caches that git ignores into ticket `ticket`'s
+ * new worktree, journaling `build_cache_skipped` for each path not copied
+ * and one `build_cache_copied` when any was.
+ *
+ * @example
+ * await copyBuildCaches({ context, ticket: 11, worktree })
+ */
+export const copyBuildCaches = async ({
+    context,
+    ticket,
+    worktree,
+}: {
+    context: BuildContext
+    ticket: number
+    worktree: string
+}): Promise<void> => {
+    const entries = buildCachePathsOf({ config: context.config })
+    if (entries.length === 0) return
+    const skip = (content: {
+        path: string
+        reason: BuildCacheSkipReason
+        error?: string
+    }) =>
+        context.journal.append({
+            kind: 'build_cache_skipped',
+            ticket,
+            role: null,
+            content,
+        })
+    // A crashed try's leftovers go first; this try's go at the end. When
+    // the temporary folder can't be cleared or made, each copy fails.
+    const temps = join(context.run_dir, 'cache-copies', String(ticket))
+    const uncleared = await rm(temps, { recursive: true, force: true }).then(
+        () => null,
+        messageOf
+    )
+    const sources = sourcesOf({ context })
+    const matches: Match[] = []
+    for (const entry of entries) {
+        const found = await matchesOf({ entry, sources })
+        if (found.length === 0) skip({ path: entry, reason: 'no_source' })
+        for (const match of found) {
+            if (!matches.some(({ path }) => path === match.path)) {
+                matches.push(match)
+            }
+        }
+    }
+    if (matches.length === 0) return
+    const ignored = await ignoredAmong({
+        git: context.git,
+        cwd: worktree,
+        paths: matches.map(({ path }) => path),
+    }).catch(() => new Set<string>())
+    const unmade =
+        uncleared ??
+        (await mkdir(temps, { recursive: true }).then(() => null, messageOf))
+    const started = performance.now()
+    const copied: { path: string; from: BuildCacheSource }[] = []
+    for (const [index, match] of matches.entries()) {
+        if (!ignored.has(match.path)) {
+            skip({ path: match.path, reason: 'not_ignored' })
+        } else if (existsSync(join(worktree, match.path))) {
+            skip({ path: match.path, reason: 'already_present' })
+        } else if (unmade !== null) {
+            skip({ path: match.path, reason: 'copy_failed', error: unmade })
+        } else {
+            const done = await copyInto({
+                context,
+                match,
+                temp: join(temps, String(index)),
+                worktree,
+            })
+            if (done.ok) copied.push({ path: match.path, from: match.from })
+            else {
+                skip({
+                    path: match.path,
+                    reason: 'copy_failed',
+                    error: done.error,
+                })
+            }
+        }
+    }
+    const ms = Math.round(performance.now() - started)
+    await removeQuietly(temps)
+    if (copied.length === 0) return
+    context.journal.append({
+        kind: 'build_cache_copied',
+        ticket,
+        role: null,
+        content: { paths: copied, ms },
+    })
+}