@vzn/vx 0.0.18 → 0.0.21

package/src/orchestrator/run.ts

@@ -3,19 +3,13 @@
 // so the layers can be swapped without touching the others.
 
 import type { ProjectEntry } from '../workspace/index.js'
-import os from 'node:os'
-import {
-  type CacheLayer,
-  type CachePolicy,
-  FULL_CACHE_POLICY,
-  type InvocationRecord,
-  type RunRecord,
-} from '../cache/index.js'
+import path from 'node:path'
+import { type CacheLayer, type CachePolicy, FULL_CACHE_POLICY } from '../cache/index.js'
 import { VERSION } from '../version.js'
-import { initSandbox, probeSandbox, resetSandbox, signalExitCode } from '../exec/index.js'
+import { resetSandbox, VX_RUN_TASK_ENV, VX_RUN_WORKSPACE_ENV } from '../exec/index.js'
 import { DeferredOutputs } from './deferred-outputs.js'
 import { resolveDownloadModes } from './download-policy.js'
-import { selectExecutor, type TaskExecutor } from '../exec/index.js'
+import type { TaskExecutor } from '../exec/index.js'
 import {
   isGroupTask,
   markSurfacedDeps,
@@ -23,22 +17,38 @@ import {
   type TaskNode,
   type TaskOutcome,
 } from '../graph/index.js'
-import { mark, MAX_TIMEOUT_MS, printTimings, ulid, UserError, editDistance } from '../util/index.js'
-import { executeTask } from './execute-task.js'
-import { resolveResourceCosts } from './resources.js'
-import { computeTaskHash } from './task-hash.js'
+import {
+  mark,
+  MAX_TIMEOUT_MS,
+  printTimings,
+  ulid,
+  nearest,
+  UserError,
+  machineParallelism,
+} from '../util/index.js'
+import { prepareSandbox } from './sandbox-request.js'
+import type { OutputDirSnapshot } from './miss-save.js'
+import { admitTasks, taintTracker } from './admission.js'
 import { busLogger, createEventBus, terminalSubscriber } from './events.js'
 import { installPlugins } from './plugin.js'
-import { resolveExecutors, teardownPlugins } from './plugin-host.js'
+import { buildAdmission, resolveExecutors, teardownPlugins } from './plugin-host.js'
 import { subscribeTelemetry, type TelemetryHandle } from './telemetry-host.js'
-import { assembleRunSummary, deriveCacheSource, isCacheHit, isPassStatus } from './telemetry.js'
-import type { RunContextRecord, TaskTelemetry } from './telemetry.js'
-import { defaultLogger, resolveOutputView, type Logger } from './logger.js'
+import { assembleRunSummary, isPassStatus } from './telemetry.js'
+import type { RunContextRecord } from './telemetry.js'
+import { defaultLogger, resolveOutputView } from './logger.js'
 import { detectColors } from './colors.js'
 import { formatPersistentList } from './framed-output.js'
 import { LocalHistoryProvider } from './history.js'
 import { plan, type RunPlan } from './plan.js'
 import { prepareRun } from './prepare.js'
+import { forwardSignals, terminateChildren } from './signals.js'
+import {
+  hasPooledExecutor,
+  placeTasks,
+  planExecutorOf,
+  poolOfPlacement,
+  UNPLACED_EXECUTOR,
+} from './placement.js'
 import {
   captureDefaultBranch,
   captureGitContext,
@@ -48,21 +58,19 @@ import {
 } from './run-context.js'
 import { startRemotePrefetch } from './remote-prefetch.js'
 import { startLocalShortCircuit, type ShortCircuit } from './local-shortcircuit.js'
-import { formatVerifySection } from './verify.js'
-
-const EMPTY_SHORT_CIRCUIT: ShortCircuit = { preProbed: new Map(), restoreTier: new Set() }
 
-/**
- * Grace after SIGTERMing the dependency-only persistent tasks at end-of-run
- * before force-killing any that trap or ignore it — so a wedged mock server
- * can't hang a normal run at completion. Well-behaved servers exit far under
- * this, so the happy path never waits it out.
- */
-const PERSISTENT_SHUTDOWN_GRACE_MS = 2000
+import { assembleRunRecords } from './run-records.js'
+import { selectKeepAlive, shutdownPersistent } from './persistent.js'
 import { writeRunProfile, writeRunSummary } from './run-artifacts.js'
-import { formatAbortedSection, formatRunSummary } from './summary.js'
+import { createSaveLane } from './save-lane.js'
+import { formatAbortedSection, formatFlakySection, formatRunSummary } from './summary.js'
+import { detectFlaky, type FlakyCandidate } from './failure-mode.js'
 import type { RunOptions, RunSummary } from './options.js'
 
+// Per run, never shared: a `vx watch` process runs many, and a shared map
+// is one `preProbed.set` away from leaking a hit across cycles.
+const emptyShortCircuit = (): ShortCircuit => ({ preProbed: new Map(), restoreTier: new Set() })
+
 /**
  * Parse the `VX_TASK_TIMEOUT` env var (ms) — the "global" run-level task
  * timeout default. A missing/empty/non-positive-integer value yields
@@ -167,7 +175,21 @@ export async function run(options: RunOptions): Promise<RunSummary> {
   const log = busLogger(bus)
 
   const prepared = await prepareRun(options, log)
-  mark('prepare (graph)')
+  mark('plugin stages')
+  // A task whose command re-enters `vx run` in the workspace running it
+  // is refused. When the inner run reaches this task again it forks a run
+  // per run until the machine gives out; when it does not (`ci` shelling
+  // out to `vx run lint`) it is a nested run the outer graph cannot see —
+  // its tasks escape the schedule, the concurrency budget and this task's
+  // cache key. The markers `taskEnv` sets on every child (exec/env.ts)
+  // name the task; the check is on the root so both shapes are caught.
+  const outerRoot = process.env[VX_RUN_WORKSPACE_ENV]
+  if (outerRoot !== undefined && path.resolve(outerRoot) === path.resolve(prepared.workspaceRoot)) {
+    prepared.cache.close()
+    throw new UserError(
+      `task ${process.env[VX_RUN_TASK_ENV] ?? '<unknown>'} runs \`vx run\` inside its own workspace: a nested run is invisible to the outer graph (its tasks escape the schedule, the concurrency budget and the cache key) and a loop back to this task forks without bound. Declare what it needs with dependsOn instead.`,
+    )
+  }
   // A requested name that matched no project is a typo (or a stray
   // positional from an `=`-only flag written with a space). Failing the
   // whole run — even when OTHER requested tasks resolved — is the point:
@@ -176,7 +198,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
   // case too; the message is identical, so that branch stays below.
   if (prepared.unresolvedTasks.length > 0) {
     log.status(
-      `No projects declare task(s): ${prepared.unresolvedTasks.join(', ')}.${didYouMean(prepared.unresolvedTasks, prepared.projects)}`,
+      `No projects declare task(s): ${prepared.unresolvedTasks.join(', ')}.${didYouMean(prepared.unresolvedTasks, prepared.projects)}${initHint(prepared)}`,
     )
     prepared.cache.close()
     return { ok: false, outcomes: [] }
@@ -188,7 +210,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     // buildTaskGraph semantics but logged just in case.
     const msg =
       prepared.empty === 'no-tasks-declared'
-        ? `No projects declare task(s): ${options.tasks.join(', ')}.`
+        ? `No projects declare task(s): ${options.tasks.join(', ')}.${initHint(prepared)}`
         : 'No tasks to run.'
     log.status(msg)
     prepared.cache.close()
@@ -197,8 +219,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
   // Install user plugins as additional bus subscribers BEFORE the run
   // starts emitting events. `installPlugins` runs each plugin's optional
   // `setup` hook and fails fast on a throw with a clean UserError naming
-  // the plugin. A plugin without `setup` (the local executor and cache)
-  // subscribes nothing.
+  // the plugin. A plugin without `setup` subscribes nothing.
   let disposePlugins: (() => void) | undefined
   let telemetry: TelemetryHandle | undefined
   try {
@@ -226,14 +247,10 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     hashCache,
     workspaceProjectCount,
   } = prepared
-  const concurrency =
-    options.concurrency ??
-    workspaceConfig?.concurrency ??
-    Math.max(1, navigator.hardwareConcurrency)
+  const concurrency = options.concurrency ?? workspaceConfig?.concurrency ?? machineParallelism()
 
-  // Resolved ONCE per run, in declaration order. A broken factory — or a
-  // workspace that declared no executor — aborts here, before any task
-  // starts.
+  // Resolved ONCE per run, in declaration order, the local executor last.
+  // A broken factory aborts here, before any task starts.
   let executors: readonly TaskExecutor[]
   try {
     executors = await resolveExecutors(prepared.plugins, {
@@ -251,13 +268,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
   // can admit a remote-pooled task against its pool instead of a local
   // worker slot. Group tasks run nothing; persistent tasks never reach an
   // executor (local by construction) — both stay off the map.
-  //
-  // `--verify=inputs` pins EVERYTHING local: the input-completeness proof
-  // is the OS sandbox, which is local machinery a remote executor silently
-  // ignores — so a remotely-executed task would pass the verify VACUOUSLY,
-  // leaky or not. A verify run is a local proof procedure by definition;
-  // determinism/fingerprint modes are unaffected (no sandbox involved).
-  const placements = placeTasks(nodes, executors, options.verify?.inputs === true)
+  const placements = placeTasks(nodes, executors, false)
   // A `remote: 'only'` task nobody takes succeeds WITHOUT running. That is
   // deliberate — on a machine with no remote pool the ambient state already
   // is what the task would have produced — but it must not be SILENT: a task
@@ -300,24 +311,8 @@ export async function run(options: RunOptions): Promise<RunSummary> {
         }
 
   // `--download` (default `all`) decides ONCE per task, here, whether a
-  // remote execution's outputs come home. `--verify` pins every task local
-  // (so nothing defers under a proof) and `all` keeps today's behaviour
-  // byte for byte.
-  // A proof must observe what it proves. `--verify=inputs` already pins
-  // every task LOCAL (so nothing could defer), but determinism and
-  // fingerprint modes do NOT — and a deferred task's outputs are absent
-  // when the verifier looks, which reported `no-outputs` for a task that
-  // declares outputs: an n/a verdict for work the proof never examined.
-  // Deferral is transfer tuning; a verify run is a rare, deliberate
-  // correctness run. Eager wins, and says so when it overrides.
-  const downloadPolicy = options.verify !== undefined ? 'all' : (options.download ?? 'all')
-  if (
-    options.verify !== undefined &&
-    options.download !== undefined &&
-    options.download !== 'all'
-  ) {
-    log.status(`vx: --verify observes outputs on disk — ignoring --download=${options.download}`)
-  }
+  // remote execution's outputs come home.
+  const downloadPolicy = options.download ?? 'all'
   const localPlaced = new Set(
     [...nodes.keys()].filter((id) => placements.executors.get(id)?.remote !== true),
   )
@@ -344,39 +339,25 @@ export async function run(options: RunOptions): Promise<RunSummary> {
   //     exit; ownership moves here so the orchestrator can SIGTERM
   //     them once the rest of the graph finishes.
   //
-  // A SIGINT/SIGTERM mid-run forwards SIGTERM to everything live,
-  // closes the cache handle, and exits 128+signo (130/143). Without
-  // this, a programmatic signal to the vx process alone (CI
-  // cancellation, `kill <pid>`) orphans every running child —
-  // terminal Ctrl-C only worked via process-group propagation. The
-  // handlers are removed in the finally below so repeated run()
-  // calls (test suites) never stack listeners.
+  // A SIGINT/SIGTERM mid-run forwards SIGTERM to everything in both
+  // (`signals.ts`); the handlers are removed in the finally below so
+  // repeated run() calls (test suites) never stack listeners.
   const liveChildren = new Set<ReturnType<typeof Bun.spawn>>()
   const persistentRegistry = new Map<string, ReturnType<typeof Bun.spawn>>()
-  const onSignal = (signal: 'SIGINT' | 'SIGTERM'): void => {
-    // Clear the live worker/status region BEFORE exiting so a TTY isn't
-    // left with a frozen region frozen in the scrollback (the documented
-    // KNOWN-OPEN). runEnd is idempotent and a no-op for non-TTY loggers.
-    try {
-      log.runEnd?.()
-    } catch {
-      // teardown must not throw on the way out
-    }
-    for (const child of liveChildren) child.kill('SIGTERM')
-    for (const child of persistentRegistry.values()) child.kill('SIGTERM')
-    try {
-      cache.close()
-    } catch {
-      // double-close race with the normal path; we're exiting anyway
-    }
-    process.exit(signalExitCode(signal))
-  }
-  const onSigint = (): void => onSignal('SIGINT')
-  const onSigterm = (): void => onSignal('SIGTERM')
-  if (options.handleSignals ?? true) {
-    process.on('SIGINT', onSigint)
-    process.on('SIGTERM', onSigterm)
+  const signals = forwardSignals({
+    enabled: options.handleSignals ?? true,
+    log,
+    cache,
+    liveChildren,
+    persistentRegistry,
+  })
+  // `RunOptions.signal`: the same teardown the process handler runs, minus
+  // the exit — the scheduler stops dispatching (it reads the signal) and
+  // run() returns to its caller. Detached in the finally below.
+  const onAbort = (): void => {
+    void terminateChildren(() => [...liveChildren, ...persistentRegistry.values()])
   }
+  options.signal?.addEventListener('abort', onAbort, { once: true })
   // The cache handle must be released on EVERY exit path, not just the
   // happy one: `close()` is also where the run's deferred `accessed_at`
   // bumps are flushed, so a throw between opening the cache and the
@@ -399,17 +380,20 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     const runStartHrTimeNs = process.hrtime.bigint()
     const endedAtMsAtStart = Date.now()
     const remoteCacheEnabled = prepared.hasRemoteLayer
-    // Normalised ONCE so every consumer — the verify gate, execute-task,
-    // the dedup predicate, the recorded invocation row — reads the policy
-    // that actually governed the run. Reading the raw request here made
-    // tasks clean their outputs before every exec for a save that never
-    // happened, and `--verify` clean them AGAIN and restore an artifact
-    // that was never written (wiping a successful build's tree and
-    // reporting it failed).
+    // Normalised ONCE so every consumer — execute-task, the dedup
+    // predicate, the recorded invocation row — reads the policy that
+    // actually governed the run. Reading the raw request here made tasks
+    // clean their outputs before every exec for a save that never
+    // happened.
     const policy: CachePolicy = effectiveCachePolicy(
       options.cache ?? FULL_CACHE_POLICY,
       prepared.hasRemoteLayer,
     )
+    if (options.remoteRequested === true && !prepared.hasRemoteLayer) {
+      log.status(
+        'vx: --cache named the remote cache, but no cache plugin in vx.workspace.* supplies one — the remote axes are off for this run',
+      )
+    }
     const deferredOutputs = new DeferredOutputs({
       nodes,
       cache,
@@ -419,29 +403,6 @@ export async function run(options: RunOptions): Promise<RunSummary> {
       localWrite: policy.localWrite,
     })
 
-    // `--verify` observes the miss-then-save path and then RESTORES attempt
-    // 1 from the artifact that save wrote, so the tree ends byte-identical
-    // to what was cached regardless of the verdict. That restore reads the
-    // LOCAL artifact file — `restoreOutputs` is a local extraction on every
-    // layer, by design — so without the local WRITE axis there is nothing to
-    // restore from: verify cleaned a successful build's declared outputs and
-    // then failed the run on the missing artifact. A remote fallback would
-    // not fix that, only narrow it to "whenever the remote is reachable at
-    // that instant", turning a guarantee into a coin flip with data loss on
-    // the losing side. So the honest gate is the local write axis, and
-    // `--no-cache` / `--cache=local:,…` / `--cache=local:r,…` are refused
-    // loudly — silently verifying nothing is the one failure mode
-    // verification must never have (same platform-honesty rule as the
-    // sandbox-unavailable error). `--force --verify` re-verifies a warm graph.
-    if (options.verify !== undefined && !policy.localWrite) {
-      throw new UserError(
-        '--verify needs the LOCAL cache write axis: it re-runs the task, then restores ' +
-          'attempt 1 from the local artifact so the outputs on disk are the ones that were ' +
-          'cached. Enable local writes (e.g. --cache=local:w,remote:rw), drop --no-cache, ' +
-          'or use --force --verify to re-execute and verify a warm graph',
-      )
-    }
-
     // Per-run context for the Tier-3 `invocations` header row. Captured
     // ONCE (git is ONE spawn for commit+branch, behind try/catch; never
     // fails a run). `dirty` reuses the `git status --porcelain` the
@@ -501,26 +462,15 @@ export async function run(options: RunOptions): Promise<RunSummary> {
       )
     }
 
-    // Lazy SRT init: fire it up if at least one task opts into sandboxing
-    // via its `sandbox: {...}` block, OR `--verify=inputs`/`=all` is on (it
-    // forces the declared-input baseline sandbox onto every cacheable task to
-    // prove input-completeness). Tasks that need sandboxing on an unsupported
-    // platform get a hard error so they don't silently run unsandboxed —
-    // `--verify=inputs` in particular must fail loud, never falsely "pass".
-    const verifyInputs = options.verify?.inputs === true
-    const anySandboxed =
-      verifyInputs || [...nodes.values()].some((n) => n.config.sandbox !== undefined)
-    if (anySandboxed) {
-      const avail = await probeSandbox()
-      if (!avail.available) {
-        throw new UserError(
-          verifyInputs
-            ? `--verify=inputs needs the sandbox, which is not available: ${avail.reason}`
-            : `sandbox not available: ${avail.reason}`,
-        )
-      }
-      await initSandbox()
-    }
+    const sandboxArmer = prepareSandbox(nodes.values())
+    const outputDirSnapshots: OutputDirSnapshot[] = []
+    // Saves run off the execution slot, twice the cap at once (memory:
+    // each pack holds an artifact's bytes); a failed save is a miss next
+    // time, said once — the task's work ran.
+    const saveLane = createSaveLane(2 * concurrency, (err) =>
+      log.status(`[vx] cache save failed: ${err instanceof Error ? err.message : String(err)}`),
+    )
+    const deferredSaves = new Map<string, Promise<void>>()
 
     // Focused flow: a requested GROUP has no output of its own, so
     // surface the same-project, non-group tasks it chains (one level)
@@ -544,16 +494,10 @@ export async function run(options: RunOptions): Promise<RunSummary> {
         if (node.requested || node.surfaced === true) requestedCount++
       }
     }
-    // Resource-aware admission: resolve every task's `exec.resources`
-    // into absolute costs ONCE, up front (percent forms against the
-    // budgets), so the scheduler's inner loop is a plain Map.get. The
-    // CPU budget is the run's concurrency; the memory budget is
-    // os.totalmem() unless `--memory` overrides it (pass `--memory` in
-    // cgroup-limited containers — totalmem() reports the HOST's RAM).
-    // Nothing declared → empty map → fields omitted from the scheduler
-    // AND the footer → byte-identical legacy path.
-    const memBudget = options.memory ?? os.totalmem()
-    const resourceCosts = resolveResourceCosts(nodes)
+    // Admission over the worker count is a plugin's (`admit`): built once
+    // into the predicate the scheduler asks at every local dispatch. No
+    // plugin answers → undefined → the count-only path, byte for byte.
+    const admit = buildAdmission(prepared.plugins, nodes, concurrency, (m) => log.status(m))
 
     // Run context for the footer. The top-of-run header is gone — the
     // banner now lives in the summary, where the eye lands at the end.
@@ -563,7 +507,6 @@ export async function run(options: RunOptions): Promise<RunSummary> {
       remoteCacheEnabled,
       concurrency,
       workspaceProjectCount,
-      ...(resourceCosts.size > 0 ? { cpuBudget: concurrency, memBudget } : {}),
     }
 
     // Lifecycle hooks drive the default logger's dynamic status line
@@ -607,7 +550,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     // each probe, so there is no second cache.get. Gated by
     // shouldShortCircuit (local reads on, no remote layer); when off, both
     // maps are empty and the run is byte-identical.
-    let shortCircuit: ShortCircuit = EMPTY_SHORT_CIRCUIT
+    let shortCircuit: ShortCircuit = emptyShortCircuit()
     if (shouldShortCircuit(nodes, policy, cache)) {
       shortCircuit = await startLocalShortCircuit({
         nodes,
@@ -622,8 +565,14 @@ export async function run(options: RunOptions): Promise<RunSummary> {
       })
     }
 
+    // Whether this task runs behind a failure (`continueMode: 'always'`
+    // only) — its save is withheld and the taint propagates; see
+    // admission.ts.
+    const isTainted = taintTracker(options.continueMode === 'always')
+
     const buildExecuteArgs = (node: TaskNode, upstream: TaskOutcome[], reuseProbe = true) => {
       const probe = reuseProbe ? shortCircuit.preProbed.get(node.id) : undefined
+      const taint = isTainted(node, upstream)
       return {
         node,
         upstream,
@@ -634,7 +583,6 @@ export async function run(options: RunOptions): Promise<RunSummary> {
         forwardArgs: options.forwardArgs,
         ...(options.retries !== undefined ? { retries: options.retries } : {}),
         ...(taskTimeoutDefault !== undefined ? { timeout: taskTimeoutDefault } : {}),
-        ...(options.verify !== undefined ? { verify: options.verify } : {}),
         log,
         executor: placements.executors.get(node.id) ?? UNPLACED_EXECUTOR,
         ...(download.modeOf.get(node.id) === 'deferred' ? { download: 'deferred' as const } : {}),
@@ -648,85 +596,39 @@ export async function run(options: RunOptions): Promise<RunSummary> {
         gitFilesCache,
         hashCache,
         ...(probe !== undefined ? { preProbed: probe } : {}),
+        ...(taint ? { taintedUpstream: true } : {}),
+        ...(sandboxArmer !== null ? { armSandbox: () => sandboxArmer.arm() } : {}),
+        outputDirSnapshots,
+        deferSave: saveLane.defer,
+        deferredSaves,
       }
     }
 
-    // In-flight dedup. Only when a service supplies a shared `inflight`
-    // registry (concurrent runs in one `vx serve`); a stateless `vx run`
-    // passes none and takes the untouched path. Gated to cacheable tasks —
-    // the join works by waiting for the sibling to populate the cache, then
-    // letting executeTask cache-hit on it. executeTask stays unchanged.
-    const inflight = options.inflight
-    const executeWithDedup = async (
-      node: TaskNode,
-      upstream: TaskOutcome[],
-    ): Promise<TaskOutcome> => {
-      // Dedup only helps when the sibling will WRITE the artifact and
-      // this task can READ it back — i.e. both axes effectively on.
-      const canRead = policy.localRead || policy.remoteRead
-      const canWrite = policy.localWrite || policy.remoteWrite
-      const cacheable =
-        !isGroupTask(node) &&
-        node.config.exec?.persistent === undefined &&
-        node.config.cache !== undefined &&
-        canRead &&
-        canWrite
-      // A restore-tier task (confirmed local hit, may run before its
-      // deps) needs no dedup — it's a restore, not an executor, and its
-      // live `upstream` is incomplete, so the dedup hash recompute would
-      // be wrong. Route it straight to executeTask, which reuses the
-      // up-front probe.
-      const restorable = shortCircuit.restoreTier.has(node.id)
-      if (inflight === undefined || !cacheable || restorable) {
-        return executeTask(buildExecuteArgs(node, upstream))
-      }
-      const hash = await computeTaskHash({
-        node,
-        upstream,
+    const executeWithDedup = admitTasks({
+      inflight: options.inflight,
+      policy,
+      shortCircuit,
+      hashArgs: {
         workspaceRoot,
         workspaceFingerprint,
         cache,
         forwardArgs: options.forwardArgs,
-        nestedProjectDirs: nestedDirsByProject.get(node.projectName) ?? [],
+        nestedDirsByProject,
         gitFilesCache,
         hashCache,
-      })
-      const existing = inflight.get(hash)
-      if (existing !== undefined) {
-        // Join a sibling already computing this exact task: wait, then
-        // executeTask cache-hits on the artifact it just saved. The
-        // up-front probe (preProbed) predates the sibling's save — its
-        // "confirmed stable miss" would skip the lazy cache.get and
-        // re-execute, defeating the dedup — so the join path drops it
-        // and lets executeTask probe fresh.
-        await existing.catch(() => {})
-        return executeTask(buildExecuteArgs(node, upstream, false))
-      }
-      // Become the executor: register a barrier siblings await. get→set has
-      // no await between, so registration is atomic — at most one executor
-      // per hash. Released on every exit (success / failure / throw).
-      let release!: () => void
-      inflight.set(
-        hash,
-        new Promise<void>((resolve) => {
-          release = resolve
-        }),
-      )
-      try {
-        return await executeTask(buildExecuteArgs(node, upstream))
-      } finally {
-        inflight.delete(hash)
-        release()
-      }
-    }
+      },
+      buildExecuteArgs,
+    })
 
     mark('classify + probe')
     const outcomes = await runGraph({
       nodes,
       concurrency,
+      settledOf: (o) => deferredSaves.get(o.node.id),
       ...(hasPooledExecutor(executors) ? { poolOf: poolOfPlacement(placements) } : {}),
-      ...(resourceCosts.size > 0 ? { resourceCosts, cpuBudget: concurrency, memBudget } : {}),
+      ...(admit !== undefined ? { admit } : {}),
       ...(options.continueMode !== undefined ? { continueMode: options.continueMode } : {}),
+      ...(options.signal !== undefined ? { signal: options.signal } : {}),
       onStart: (node) => {
         log.taskStart?.(node)
       },
@@ -744,72 +646,22 @@ export async function run(options: RunOptions): Promise<RunSummary> {
       restoreTier: shortCircuit.restoreTier,
     })
 
-    // A persistent task the user REQUESTED (a dev server / watcher) is
-    // the run's whole purpose — don't tear it down the instant it's
-    // ready. We leave those running and block on them at the very end
-    // (after the normal summary prints); everything else (persistent
-    // tasks pulled in only as dependencies of now-finished work) is
-    // SIGTERMed here as before. Scoped to the real CLI foreground:
-    // `options.log === undefined` means the default logger (a `vx run`
-    // invocation), and `handleSignals` excludes watch mode (own signal
-    // loop) and embedders that manage lifecycle themselves — both expect
-    // run() to return, not block on a server.
+    // Which persistent children outlive the graph, and the bounded SIGTERM
+    // of the rest, before the summary prints. Scoped to the real CLI
+    // foreground: `options.log === undefined` means the default logger (a
+    // `vx run` invocation), and `handleSignals` excludes watch mode (own
+    // signal loop) and embedders that manage lifecycle themselves — both
+    // expect run() to return, not block on a server.
     const foreground = options.log === undefined && (options.handleSignals ?? true)
-    const keepAliveNodes: TaskNode[] = []
-    const keepAlive: ReturnType<typeof Bun.spawn>[] = []
-    if (foreground) {
-      for (const [id, child] of persistentRegistry) {
-        const n = nodes.get(id)
-        if (n !== undefined && (n.requested || n.surfaced === true)) {
-          keepAliveNodes.push(n)
-          keepAlive.push(child)
-        }
-      }
-    }
-    const keepAliveSet = new Set(keepAlive)
-
-    // Shut down the dependency-only persistent tasks before reporting the final
-    // summary. SIGTERM gives well-behaved servers (vite, next, esbuild --watch)
-    // a moment to clean up. Bun's Subprocess.kill is idempotent on an
-    // already-exited child. Bound the wait: a persistent dep that traps or
-    // ignores SIGTERM (a wedged mock server) would otherwise hang the run at
-    // NORMAL completion forever — after a grace, SIGKILL the stragglers and move
-    // on. Well-behaved servers exit in well under the grace, so the happy path
-    // pays nothing; the timer is cleared + unref'd so a fast shutdown never
-    // delays CLI exit.
-    const dyingChildren = [...persistentRegistry.values()].filter((c) => !keepAliveSet.has(c))
-    for (const child of dyingChildren) child.kill('SIGTERM')
-    const allExited = Promise.allSettled(dyingChildren.map((c) => c.exited))
-    let graceTimer: ReturnType<typeof setTimeout> | undefined
-    const winner = await Promise.race([
-      allExited.then(() => 'exited' as const),
-      new Promise<'grace'>((resolve) => {
-        graceTimer = setTimeout(() => resolve('grace'), PERSISTENT_SHUTDOWN_GRACE_MS)
-        graceTimer.unref?.()
-      }),
-    ])
-    if (graceTimer !== undefined) clearTimeout(graceTimer)
-    if (winner === 'grace') {
-      for (const child of dyingChildren) child.kill('SIGKILL')
-      await allExited
-    }
+    const keepAlive = selectKeepAlive(persistentRegistry, nodes, foreground)
+    await shutdownPersistent(persistentRegistry, keepAlive.children)
 
     mark('run graph')
     // Clear the status line for good before the summary prints.
     log.runEnd?.()
 
     const list = [...outcomes.values()]
-    const ok =
-      list.every((o) => isPassStatus(o.status)) &&
-      // `--verify`: a provably-unsafe cache entry (non-deterministic outputs, a
-      // re-run that failed, or a read of undeclared inputs) turns the run red so
-      // CI catches it.
-      !list.some(
-        (o) =>
-          o.verify?.kind === 'nondeterministic' ||
-          o.verify?.kind === 'rerun-failed' ||
-          o.verify?.kind === 'undeclared-inputs',
-      )
+    const ok = list.every((o) => isPassStatus(o.status))
 
     // The summary + artifact writers + recordRun pass all exclude group
     // tasks via the shared tallyOutcomes helper. We pass the full
@@ -818,13 +670,18 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     const totalMs = Number(process.hrtime.bigint() - runStartHrTimeNs) / 1_000_000
     // Foreground dev mode: between the task frame and the footer, list
     // the persistent tasks still running (see the keep-alive block below).
-    if (keepAliveNodes.length > 0) {
-      for (const line of formatPersistentList(keepAliveNodes, colors)) log.status(line)
+    if (keepAlive.nodes.length > 0) {
+      for (const line of formatPersistentList(keepAlive.nodes, colors)) log.status(line)
     }
     for (const line of formatRunSummary(list, totalMs, colors, runContext)) log.status(line)
     // A task killed by a shutdown signal is in no bucket above, yet it makes
     // `ok` false — name it, or the red exit is undiagnosable.
     for (const line of formatAbortedSection(list)) log.status(line)
+    // Judged against the history BEFORE this run's rows land, so the query
+    // is one scan over the executed tasks' keys and nothing at all on a run
+    // that executed none (every hit, every skip).
+    const flaky = detectFlaky(prepared.localCache.dbHandle(), flakyCandidates(list))
+    for (const line of formatFlakySection(flaky)) log.status(line)
     // Outputs that never came home are not an error, but a silent `dist/`
     // that is empty-or-stale would be: name every task whose bytes are
     // still remote.
@@ -835,24 +692,6 @@ export async function run(options: RunOptions): Promise<RunSummary> {
         `  Deferred: ${stillDeferred.length} task(s) left outputs remote (--download=none): ${stillDeferred.join(', ')}`,
       )
     }
-    if (options.verify !== undefined) {
-      for (const line of formatVerifySection(list)) log.status(line)
-      // A fingerprint-only run attaches no verdicts, so the verdict-driven
-      // section above prints nothing — report what actually happened.
-      if (options.verify.fingerprint && !options.verify.determinism && !options.verify.inputs) {
-        const n = list.filter((o) => o.outputFp !== undefined).length
-        log.status('')
-        log.status(
-          `  Verify:   fingerprinted ${n} task output trees (cross-machine diff via a connected serve)`,
-        )
-        // Only EXECUTED tasks fingerprint — a warm all-hit run reports 0.
-        // A per-platform matrix wired without `--force` produces nothing
-        // forever, so name the cause instead of a bare 0.
-        if (n === 0) {
-          log.status('            (0 executed — cache hits do not fingerprint; pair with --force)')
-        }
-      }
-    }
 
     // Optional artifacts. Errors are surfaced to the user but don't
     // change the run's exit code — the run already happened.
@@ -868,6 +707,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
           totalMs,
           ok,
           outcomes: list,
+          flaky,
         })
         log.status(`vx: summary written to ${wrote}`)
       } catch (err) {
@@ -890,119 +730,32 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     }
 
     // Record each task to the run history in a single SQLite transaction
-    // (one fsync instead of N). Group tasks (no `exec`) are skipped —
-    // they aren't real runs and the `runs` table is analytics-focused.
-    // One invocation header row is written alongside, atomically via
-    // recordRunBundle. The Tier-3 input-fingerprint rows (entry_inputs)
-    // are NOT built here — they're persisted inside each entry's save
-    // transaction (miss path only), so a warm all-cache-hit run does no
-    // extra recording work.
-    const now = endedAtMs
-    const toRecord: RunRecord[] = []
-    // Per-task telemetry mirrors `toRecord` 1:1 — built only when a
-    // telemetry sink is active, so a no-telemetry run allocates nothing.
-    const summaryTasks: TaskTelemetry[] = []
-    let failedCount = 0
-    let hitLocalCount = 0
-    let hitRemoteCount = 0
-    for (const o of list) {
-      if (isGroupTask(o.node)) continue
-      // aborted (killed by a shutdown signal) isn't a real run.
-      if (o.status === 'aborted') continue
-      if (telemetry !== undefined) {
-        const t: TaskTelemetry = {
-          taskId: o.node.id,
-          project: o.node.projectName,
-          task: o.node.taskName,
-          status: o.status,
-          cacheSource: deriveCacheSource(o.status),
-          exitCode: o.exitCode,
-          durationMs: o.durationMs,
-        }
-        if (o.hash !== undefined) t.hash = o.hash
-        if (o.cpuMs !== undefined) t.cpuMs = o.cpuMs
-        if (o.peakRssBytes !== undefined) t.peakRssBytes = o.peakRssBytes
-        if (o.where !== undefined) t.where = o.where
-        if (o.outputs !== undefined) t.outputs = o.outputs
-        if (o.attempts !== undefined) t.attempts = o.attempts
-        if (o.verify !== undefined) t.verify = o.verify
-        if (o.outputFp !== undefined) t.outputFp = o.outputFp
-        if (o.wallclockStartNs !== undefined) t.wallclockStartNs = o.wallclockStartNs.toString()
-        if (o.wallclockEndNs !== undefined) t.wallclockEndNs = o.wallclockEndNs.toString()
-        summaryTasks.push(t)
-      }
-      // Every non-group, non-aborted outcome gets a row — the same set
-      // `tallyOutcomes` counts, so `invocations.task_count` equals both the
-      // terminal's "N total" and `COUNT(*) FROM runs WHERE run_id = ?`.
-      // An outcome with NO hash (a `skipped` task never probed the cache; a
-      // `persistent` one is never cacheable) used to be dropped here because
-      // `runs.hash` is NOT NULL — which made a failing persistent task record
-      // `0 tasks, 0 failures` on a run the terminal called red, and a failed
-      // task with a skipped dependent record 1 of 2. `bindRun` stores `''` for
-      // those instead; the key-diff readers guard it.
-      toRecord.push({
-        ...(o.hash !== undefined ? { hash: o.hash } : {}),
-        project: o.node.projectName,
-        task: o.node.taskName,
-        status: o.status,
-        exitCode: o.exitCode,
-        durationMs: o.durationMs,
-        ...(options.forwardArgs !== undefined ? { forwardArgs: options.forwardArgs } : {}),
-        // Anchor to the REAL per-task wall-clock window: run-start wall time +
-        // the task's ns offset (captured for hits and executed tasks alike).
-        // The `now - duration` fallback applies to outcomes without an offset —
-        // today only `skipped`, which the scheduler finishes synchronously with
-        // no span, so it collapses to a zero-width mark at the run's end. Using
-        // run-end-minus-duration for EVERYTHING was the old bug that piled every
-        // task at the right edge of the timeline.
-        startedAt:
-          o.wallclockStartNs !== undefined
-            ? endedAtMsAtStart + Math.round(Number(o.wallclockStartNs) / 1e6)
-            : now - o.durationMs,
-        endedAt:
-          o.wallclockEndNs !== undefined
-            ? endedAtMsAtStart + Math.round(Number(o.wallclockEndNs) / 1e6)
-            : now,
-        runId,
-        ...(o.cpuMs !== undefined ? { cpuMs: o.cpuMs } : {}),
-        ...(o.peakRssBytes !== undefined ? { peakRssBytes: o.peakRssBytes } : {}),
-        ...(o.wallclockStartNs !== undefined ? { wallclockStartNs: o.wallclockStartNs } : {}),
-        ...(o.wallclockEndNs !== undefined ? { wallclockEndNs: o.wallclockEndNs } : {}),
-        cacheHit: isCacheHit(o.status),
-        ...(o.attempts !== undefined ? { attempts: o.attempts } : {}),
-      })
-      if (o.status === 'failed') failedCount++
-      if (o.status === 'cache-hit') hitLocalCount++
-      if (o.status === 'cache-hit-remote') hitRemoteCount++
-    }
-    const invocation: InvocationRecord = {
+    // (one fsync instead of N), with the invocation header row alongside,
+    // atomically via recordRunBundle. The Tier-3 input-fingerprint rows
+    // (entry_inputs) are NOT built here — they're persisted inside each
+    // entry's save transaction (miss path only), so a warm all-cache-hit
+    // run does no extra recording work. The telemetry mirror is built in
+    // the same pass, only when a sink is active.
+    const records = assembleRunRecords({
+      outcomes: list,
       runId,
+      startedAtMs: endedAtMsAtStart,
+      endedAtMs,
+      totalMs,
+      ok,
       command: options.command ?? process.argv.slice(1).join(' '),
-      requestedTasks: JSON.stringify([...options.tasks]),
+      requestedTasks: options.tasks,
       cachePolicy: compactCachePolicy(policy),
       concurrency,
       flow: options.flow ?? null,
-      startedAt: endedAtMsAtStart,
-      endedAt: endedAtMs,
-      totalDurationMs: Math.round(totalMs),
-      taskCount: toRecord.length,
-      failedCount,
-      hitCount: hitLocalCount + hitRemoteCount,
-      hitLocalCount,
-      hitRemoteCount,
-      exitOk: ok,
-      commitSha: gitContext.commitSha,
-      branch: gitContext.branch,
-      dirty: gitContext.dirty,
-      ci: ciContext.ci,
-      ciProvider: ciContext.provider,
-      host: hostContext.host,
-      os: hostContext.os,
-      arch: hostContext.arch,
-      vxVersion: VERSION,
-      tags: JSON.stringify(options.tags ?? {}),
-    }
-    cache.recordRunBundle({ runs: toRecord, invocation })
+      forwardArgs: options.forwardArgs,
+      tags: options.tags ?? {},
+      git: gitContext,
+      ci: ciContext,
+      host: hostContext,
+      withTelemetry: telemetry !== undefined,
+    })
+    cache.recordRunBundle(records)
     mark('record history')
     // Hand the per-run summary to the telemetry sinks + drain them. Only
     // when a sink is active (telemetry !== undefined) — otherwise this
@@ -1010,7 +763,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     // emitSummary/flush are crash-isolated, so a faulty sink can't fail
     // the run; flush is the sink's last chance to ship buffered records.
     if (telemetry !== undefined && runContextRecord !== undefined) {
-      const summary = assembleRunSummary(runContextRecord, summaryTasks, {
+      const summary = assembleRunSummary(runContextRecord, records.telemetryTasks, {
         startedAt: endedAtMsAtStart,
         endedAt: endedAtMs,
         totalDurationMs: Math.round(totalMs),
@@ -1039,7 +792,24 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     // write with nothing but a warning. The seam's contract is that a
     // cache layer stays usable until the run's uploads have settled.
     await prefetchDone
+    // Every deferred save settles first: the upload drain below carries
+    // the entries the saves queued, and the snapshot loop after it reads
+    // what the saves pushed.
+    await saveLane.drain()
+    mark('save lane')
     await cache.drainUploads?.()
+    // The miss path's output-directory snapshots, taken now that the
+    // directories are old enough for the snapshot's racy window (see
+    // miss-save.ts). A few at a time: each is an lstat + readdir per
+    // prefix and one index transaction.
+    for (let i = 0; i < outputDirSnapshots.length; i += 32) {
+      await Promise.all(
+        outputDirSnapshots
+          .slice(i, i + 32)
+          .map((s) => cache.recordOutputDirs?.(s.hash, s.projectDir, s.prefixes)),
+      )
+    }
+    mark('output dir snapshots')
     await teardownPlugins(prepared.plugins, (m) => log.status(m))
     closeCache()
     mark('close')
@@ -1048,7 +818,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     // Tear down SRT's network bridge + (on macOS) log monitor. No-op if
     // no task was sandboxed; otherwise SRT keeps proxy servers alive and
     // the next vx run would init on top of stale state.
-    if (anySandboxed) {
+    if (sandboxArmer?.armed) {
       try {
         await resetSandbox()
       } catch (err) {
@@ -1061,12 +831,30 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     // persistent task (dev server / watcher). The run is "done" in every
     // bookkeeping sense — summary printed, history recorded — but the
     // server is still up and that's the point. Stay in the foreground
-    // until it exits: Ctrl-C hits the whole process group (the server
-    // dies; our SIGINT handler also exits 130), and a crash resolves the
-    // wait so the run returns. Nothing here prints — the UI is unchanged.
-    if (keepAlive.length > 0) {
-      await Promise.allSettled(keepAlive.map((c) => c.exited))
-      for (const child of keepAlive) child.kill('SIGTERM')
+    // until ONE of them exits: Ctrl-C hits the whole process group (the
+    // server dies; our SIGINT handler also exits 130), and a crash ends
+    // the session — the others are torn down (SIGTERM, grace, SIGKILL)
+    // and a non-zero exit makes the run not ok, so `vx run dev` in a
+    // script fails when the server it started fell over. Until
+    // 2026-09-10 this waited for EVERY server, so the SIGTERM after it
+    // was dead code and a crashed server left the rest running under a
+    // run that never returned. One status line names the server that
+    // ended the session and its code — the summary above said `success`
+    // for it, and an exit 1 with no word about why is a mystery in a log.
+    if (keepAlive.children.length > 0) {
+      const first = await Promise.race(
+        keepAlive.children.map((c, i) => c.exited.then((code) => ({ code, i }))),
+      )
+      const node = keepAlive.nodes[first.i]!
+      const others = keepAlive.nodes.length - 1
+      log.status(
+        `vx: ${node.id} exited with code ${first.code}` +
+          (others > 0
+            ? `; stopping ${others} other persistent task${others === 1 ? '' : 's'}`
+            : ''),
+      )
+      await terminateChildren(() => keepAlive.children)
+      return { ok: ok && first.code === 0, outcomes: list }
     }
 
     return { ok, outcomes: list }
@@ -1074,8 +862,8 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     // Idempotent; also reached on mid-run throws, so a crashed cycle
     // can't leave a live status-line ticker behind.
     log.runEnd?.()
-    process.off('SIGINT', onSigint)
-    process.off('SIGTERM', onSigterm)
+    signals.remove()
+    options.signal?.removeEventListener('abort', onAbort)
     // Plugins installed at the top of run() get their bus subscriptions
     // released here. Idempotent; safe even if installPlugins threw.
     disposePlugins?.()
@@ -1140,172 +928,23 @@ export async function planRun(options: RunOptions): Promise<RunPlan> {
     })
   } finally {
     prepared.cache.close()
+    // The same table a run prints: a dry run is how the prepare stages
+    // (discovery, config load, the git enumeration) get profiled on a
+    // real repo that has no install and no cache to run against.
+    mark('close')
+    printTimings()
   }
 }
 
 /**
- * A task is pinned to this machine when it is persistent, transitively
- * depends on a persistent task (a worker cannot reach a port on the
- * submitter), or declares `exec.remote: false`.
- */
-function pinnedLocalSet(nodes: Map<string, TaskNode>): Set<string> {
-  const pinned = new Set<string>()
-  const memo = new Map<string, boolean>()
-  const visit = (id: string): boolean => {
-    const known = memo.get(id)
-    if (known !== undefined) return known
-    const node = nodes.get(id)
-    if (node === undefined) return false
-    memo.set(id, false) // cycle guard; the graph builder already rejects cycles
-    const result =
-      node.config.exec?.persistent !== undefined ||
-      node.config.exec?.remote === false ||
-      node.deps.some((d) => visit(d))
-    memo.set(id, result)
-    if (result) pinned.add(id)
-    return result
-  }
-  for (const id of nodes.keys()) visit(id)
-  return pinned
-}
-
-interface Placements {
-  executors: Map<string, TaskExecutor>
-  /**
-   * `exec.remote: 'only'` tasks that no REMOTE executor took — a NO-OP on
-   * this machine: never executed, declared outputs never cleaned or
-   * restored. The task exists to produce a remote input tree; without a
-   * remote pool, dependents use the machine's ambient state exactly as they
-   * did before the field existed.
-   */
-  remoteOnlyNoop: Set<string>
-  /** `'only'` tasks a remote executor DID take — executed remotely, outputs stay remote. */
-  remoteOnly: Set<string>
-}
-
-function placeTasks(
-  nodes: Map<string, TaskNode>,
-  executors: readonly TaskExecutor[],
-  pinAllLocal = false,
-): Placements {
-  const pinned = pinnedLocalSet(nodes)
-  const placements: Placements = {
-    executors: new Map(),
-    remoteOnlyNoop: new Set(),
-    remoteOnly: new Set(),
-  }
-  for (const node of nodes.values()) {
-    if (isGroupTask(node) || node.config.exec?.persistent !== undefined) continue
-    const executor = selectExecutor(executors, {
-      taskId: node.id,
-      projectName: node.projectName,
-      projectDir: node.projectDir,
-      command: node.config.exec!.command,
-      pinnedLocal: pinAllLocal || pinned.has(node.id),
-      cacheable: node.config.cache !== undefined,
-      ...(node.config.exec?.resources === undefined
-        ? {}
-        : { resources: node.config.exec.resources }),
-    })
-    placements.executors.set(node.id, executor)
-    if (node.config.exec?.remote === 'only') {
-      // A pinned 'only' task (it transitively depends on a persistent one)
-      // lands here too: pinning wins, so it noops rather than shipping.
-      if (executor.remote === true) placements.remoteOnly.add(node.id)
-      else placements.remoteOnlyNoop.add(node.id)
-    }
-  }
-  return placements
-}
-
-/**
- * `executorOf` for `planRun`, or nothing. Declining plugins, a single
- * executor, or a resolution error all yield nothing: `--dry` is an
- * inspection command and must not fail over a label.
- */
-async function planExecutorOf(
-  prepared: Awaited<ReturnType<typeof prepareRun>>,
-  log: Logger,
-  policy: 'all' | 'toplevel' | 'none',
-): Promise<{
-  executorOf?: (id: string) => string | undefined
-  downloadOf?: (id: string) => 'eager' | 'deferred' | 'never' | undefined
-  downloadDowngrades?: ReadonlyArray<{ taskId: string; reason: string }>
-}> {
-  let executors: readonly TaskExecutor[]
-  try {
-    executors = await resolveExecutors(prepared.plugins, {
-      workspaceRoot: prepared.workspaceRoot,
-      cacheDir: prepared.cacheDir,
-      warn: (m: string) => log.status(m),
-      concurrency: Math.max(1, navigator.hardwareConcurrency),
-    })
-  } catch {
-    return {}
-  }
-  const placements = placeTasks(prepared.nodes, executors)
-  // Download modes need placement regardless of how many executors there
-  // are (a single REMOTE one still defers); executor LABELS only earn their
-  // column when there is a choice to report.
-  const download =
-    policy === 'all'
-      ? undefined
-      : resolveDownloadModes({
-          nodes: prepared.nodes,
-          policy,
-          localPlaced: new Set(
-            [...prepared.nodes.keys()].filter(
-              (id) => placements.executors.get(id)?.remote !== true,
-            ),
-          ),
-          remoteOnly: placements.remoteOnly,
-        })
-  return {
-    ...(executors.length < 2
-      ? {}
-      : {
-          executorOf: (id: string) =>
-            placements.remoteOnlyNoop.has(id) ? 'noop' : placements.executors.get(id)?.name,
-        }),
-    ...(download === undefined
-      ? {}
-      : {
-          downloadOf: (id: string) => download.modeOf.get(id),
-          downloadDowngrades: [...download.downgrades].map(([taskId, reason]) => ({
-            taskId,
-            reason,
-          })),
-        }),
-  }
-}
-
-/**
- * Stands in for the executor of a task that was never placed — a group task
- * (runs nothing) or a persistent one (`executePersistentTask` owns it and
- * never reads this field). It THROWS rather than silently picking some
- * executor from the list, so a refactor that routes such a task through the
- * exec path fails loudly instead of shipping a localhost server to a worker.
+ * The first-run case, told apart from a typo: no package in the workspace
+ * has a `vx.config.*` at all, so no name could have resolved. Reached only
+ * on the error path.
  */
-const UNPLACED_EXECUTOR: TaskExecutor = {
-  name: 'unplaced',
-  execute: (req) => {
-    throw new Error(`internal error: ${req.taskId} reached an executor without being placed`)
-  },
-}
-
-function hasPooledExecutor(executors: readonly TaskExecutor[]): boolean {
-  return executors.some((e) => e.capacity !== undefined)
-}
-
-function poolOfPlacement(
-  placements: Placements,
-): (id: string) => { name: string; capacity: number } | undefined {
-  return (id) => {
-    const executor = placements.executors.get(id)
-    return executor?.capacity === undefined
-      ? undefined
-      : { name: executor.name, capacity: executor.capacity }
-  }
+function initHint(prepared: { anyProjectConfig: boolean }): string {
+  return prepared.anyProjectConfig
+    ? ''
+    : ' No package declares a vx.config — run `vx init` to write one per package from its package.json scripts.'
 }
 
 /**
@@ -1319,37 +958,47 @@ function didYouMean(
   unresolved: readonly string[],
   projects: ReadonlyMap<string, ProjectEntry>,
 ): string {
-  const nearest = (name: string, candidates: Iterable<string>): string | undefined => {
-    let best: string | undefined
-    let bestD = 3
-    for (const c of candidates) {
-      const d = editDistance(name, c)
-      if (d < bestD) {
-        bestD = d
-        best = c
-      }
-    }
-    return best === name ? undefined : best
-  }
   const tasksOf = (p: ProjectEntry | undefined): string[] => Object.keys(p?.config.tasks ?? {})
   const allTasks = new Set<string>()
   for (const p of projects.values()) for (const t of tasksOf(p)) allTasks.add(t)
-  const hints: string[] = []
+  // A Set: two typos of the same task hint it once, not once per typo.
+  const hints = new Set<string>()
   for (const spec of unresolved) {
     const at = spec.indexOf('#')
     if (at < 0) {
       const t = nearest(spec, allTasks)
-      if (t !== undefined) hints.push(t)
+      if (t !== undefined) hints.add(t)
       continue
     }
     const [proj, task] = [spec.slice(0, at), spec.slice(at + 1)]
     if (!projects.has(proj)) {
       const p = nearest(proj, projects.keys())
-      if (p !== undefined && tasksOf(projects.get(p)).includes(task)) hints.push(`${p}#${task}`)
+      if (p !== undefined && tasksOf(projects.get(p)).includes(task)) hints.add(`${p}#${task}`)
       continue
     }
     const t = nearest(task, tasksOf(projects.get(proj)))
-    if (t !== undefined) hints.push(`${proj}#${t}`)
+    if (t !== undefined) hints.add(`${proj}#${t}`)
+  }
+  return hints.size === 0 ? '' : ` Did you mean ${[...hints].join(', ')}?`
+}
+
+/** The executed, keyed outcomes of a run — what flakiness is judged on. */
+function flakyCandidates(outcomes: readonly TaskOutcome[]): FlakyCandidate[] {
+  const out: FlakyCandidate[] = []
+  for (const o of outcomes) {
+    if (o.status !== 'success' && o.status !== 'failed') continue
+    // "Same inputs, different outcome" is a claim only a task with declared
+    // inputs can make: a task with no `cache` block keys on its config alone
+    // and runs every time, so one bad network day would read as a flake for
+    // thirty days. Groups do no work.
+    if (o.hash === undefined || o.node.config.cache === undefined || isGroupTask(o.node)) continue
+    out.push({
+      project: o.node.projectName,
+      task: o.node.taskName,
+      hash: o.hash,
+      status: o.status,
+      attempts: o.attempts ?? 1,
+    })
   }
-  return hints.length === 0 ? '' : ` Did you mean ${hints.join(', ')}?`
+  return out
 }