@vzn/vx 0.0.15 → 0.0.18

package/src/orchestrator/run.ts

@@ -2,17 +2,20 @@
 // run with caching. Each step delegates to a single-purpose sibling file
 // so the layers can be swapped without touching the others.
 
+import type { ProjectEntry } from '../workspace/index.js'
 import os from 'node:os'
 import {
   type CacheLayer,
   type CachePolicy,
   FULL_CACHE_POLICY,
   type InvocationRecord,
-  LayeredCache,
   type RunRecord,
 } from '../cache/index.js'
 import { VERSION } from '../version.js'
 import { initSandbox, probeSandbox, resetSandbox, signalExitCode } from '../exec/index.js'
+import { DeferredOutputs } from './deferred-outputs.js'
+import { resolveDownloadModes } from './download-policy.js'
+import { selectExecutor, type TaskExecutor } from '../exec/index.js'
 import {
   isGroupTask,
   markSurfacedDeps,
@@ -20,24 +23,24 @@ import {
   type TaskNode,
   type TaskOutcome,
 } from '../graph/index.js'
-import { ulid, UserError } from '../util/index.js'
+import { mark, MAX_TIMEOUT_MS, printTimings, ulid, UserError, editDistance } from '../util/index.js'
 import { executeTask } from './execute-task.js'
 import { resolveResourceCosts } from './resources.js'
 import { computeTaskHash } from './task-hash.js'
 import { busLogger, createEventBus, terminalSubscriber } from './events.js'
 import { installPlugins } from './plugin.js'
-import type { VxPlugin } from './plugin.js'
-import { subscribeEventSinks, teardownPlugins } from './plugin-host.js'
-import type { SubscribedEventSinks } from './plugin-host.js'
+import { resolveExecutors, teardownPlugins } from './plugin-host.js'
 import { subscribeTelemetry, type TelemetryHandle } from './telemetry-host.js'
-import { deriveCacheSource, TELEMETRY_SCHEMA_VERSION } from './telemetry.js'
-import type { RunContextRecord, RunSummaryRecord, TaskTelemetry } from './telemetry.js'
-import { defaultLogger, resolveOutputView } from './logger.js'
+import { assembleRunSummary, deriveCacheSource, isCacheHit, isPassStatus } from './telemetry.js'
+import type { RunContextRecord, TaskTelemetry } from './telemetry.js'
+import { defaultLogger, resolveOutputView, type Logger } from './logger.js'
 import { detectColors } from './colors.js'
 import { formatPersistentList } from './framed-output.js'
+import { LocalHistoryProvider } from './history.js'
 import { plan, type RunPlan } from './plan.js'
 import { prepareRun } from './prepare.js'
 import {
+  captureDefaultBranch,
   captureGitContext,
   captureHostContext,
   captureWorkspaceIdentity,
@@ -48,21 +51,38 @@ import { startLocalShortCircuit, type ShortCircuit } from './local-shortcircuit.
 import { formatVerifySection } from './verify.js'
 
 const EMPTY_SHORT_CIRCUIT: ShortCircuit = { preProbed: new Map(), restoreTier: new Set() }
+
+/**
+ * Grace after SIGTERMing the dependency-only persistent tasks at end-of-run
+ * before force-killing any that trap or ignore it — so a wedged mock server
+ * can't hang a normal run at completion. Well-behaved servers exit far under
+ * this, so the happy path never waits it out.
+ */
+const PERSISTENT_SHUTDOWN_GRACE_MS = 2000
 import { writeRunProfile, writeRunSummary } from './run-artifacts.js'
-import { formatRunSummary } from './summary.js'
+import { formatAbortedSection, formatRunSummary } from './summary.js'
 import type { RunOptions, RunSummary } from './options.js'
 
 /**
  * Parse the `VX_TASK_TIMEOUT` env var (ms) — the "global" run-level task
  * timeout default. A missing/empty/non-positive-integer value yields
  * `undefined` (ignored), so a typo never silently disables a task's own
- * `exec.timeout`.
+ * `exec.timeout`. A value past `MAX_TIMEOUT_MS` is clamped to it — see below.
+ *
+ * Exported for `tests/options-resolve.test.ts`, which pins every accepted
+ * and ignored form of this rung; it has no other caller outside this file.
  */
-function readTaskTimeoutEnv(): number | undefined {
+export function readTaskTimeoutEnv(): number | undefined {
   const raw = process.env['VX_TASK_TIMEOUT']
   if (raw === undefined || raw === '') return undefined
   const n = Number(raw)
-  return Number.isInteger(n) && n > 0 ? n : undefined
+  if (!Number.isInteger(n) || n <= 0) return undefined
+  // CLAMPED, not refused — unlike `exec.timeout` and `--timeout`, this rung's
+  // contract is already "unusable value → fall back", so it never throws. But
+  // falling back is wrong here: someone who typed a huge number wants a long
+  // timeout, and the largest expressible one IS long (~24.8 days). Handing it
+  // to `setTimeout` unbounded would mean 1 ms — killing every task instantly.
+  return Math.min(n, MAX_TIMEOUT_MS)
 }
 
 /**
@@ -79,32 +99,49 @@ function compactCachePolicy(p: CachePolicy): string {
   return parts.join(',')
 }
 
+/**
+ * The policy that will actually govern the run. The remote axes are inert
+ * without a remote layer to serve them: a `remote:w` policy over a bare
+ * local cache writes NOWHERE, yet the write axis reads as on. Normalising
+ * here — the ONE place both `run()` and `planRun()` derive it from — is
+ * what keeps `--dry` describing the run you are about to get: reading the
+ * raw request made the plan label `--cache=local:,remote:rw` "cache miss —
+ * would exec" (a result that will be stored) for a run in which caching is
+ * entirely off.
+ */
+function effectiveCachePolicy(requested: CachePolicy, hasRemoteLayer: boolean): CachePolicy {
+  return hasRemoteLayer ? requested : { ...requested, remoteRead: false, remoteWrite: false }
+}
+
 /**
  * The perf firewall for the local short-circuit. Always-on; the only
  * gates are correctness/no-op gates:
- *   - LOCAL-ONLY cache. Under a LayeredCache, `cache.get` is a remote
+ *   - LOCAL-ONLY cache. Behind a remote layer, `cache.get` is a remote
  *     READ-THROUGH — the up-front classify is awaited before scheduling,
  *     so N remote GETs would land on the critical path before any task
  *     starts. Remote runs are owned by `startRemotePrefetch`, which
  *     overlaps the GETs with execution instead (fire-and-forget) and
- *     ingests hits into local for execute-task's lazy probe.
+ *     ingests hits into local for execute-task's lazy probe. The gate asks
+ *     the layer (`hasRemote`), not `instanceof LayeredCache` — a
+ *     third-party remote layer stalls on the classify just as hard.
  *   - the policy reads locally (a `--no-cache`/`--force`/`--cache=local:`
  *     run reads nothing locally → nothing to restore → skip);
- *   - the graph has at least one dependency edge (a flat graph has no
- *     ordering to bypass — restoring is what execute() already does, so
- *     there's no upside and we avoid the upfront probe pass).
- * The per-task correctness gates (stable key, no workspace-outputs in
- * graph) live in `startLocalShortCircuit`.
+ *   - there is at least one task.
+ * A flat graph has no ordering to bypass, but the classify pass is worth
+ * paying for anyway: it probes every stable key through ONE batched
+ * `getMany` instead of a `cache.get` per task inside the run, which
+ * measured 84 → 78 ms on 100 dep-free tasks and 249 → 237 ms on 1000
+ * (2026-09-03, interleaved arms). The per-task correctness gates (stable
+ * key, no workspace-outputs in graph) live in `startLocalShortCircuit`.
  */
 export function shouldShortCircuit(
   nodes: Map<string, TaskNode>,
   policy: CachePolicy,
   cache: CacheLayer,
 ): boolean {
-  if (cache instanceof LayeredCache) return false
+  if (cache.hasRemote === true) return false
   if (!policy.localRead) return false
-  for (const node of nodes.values()) if (node.deps.length > 0) return true
-  return false
+  return nodes.size > 0
 }
 
 export async function run(options: RunOptions): Promise<RunSummary> {
@@ -130,6 +167,20 @@ export async function run(options: RunOptions): Promise<RunSummary> {
   const log = busLogger(bus)
 
   const prepared = await prepareRun(options, log)
+  mark('prepare (graph)')
+  // A requested name that matched no project is a typo (or a stray
+  // positional from an `=`-only flag written with a space). Failing the
+  // whole run — even when OTHER requested tasks resolved — is the point:
+  // a CI job that renames a task must go red, not silently stop running
+  // it. When EVERY name is unresolved this is the `no-tasks-declared`
+  // case too; the message is identical, so that branch stays below.
+  if (prepared.unresolvedTasks.length > 0) {
+    log.status(
+      `No projects declare task(s): ${prepared.unresolvedTasks.join(', ')}.${didYouMean(prepared.unresolvedTasks, prepared.projects)}`,
+    )
+    prepared.cache.close()
+    return { ok: false, outcomes: [] }
+  }
   if (prepared.empty !== null) {
     // `no-tasks-declared` is almost always a typo in CI; we surface
     // a clear message and return NOT-ok so the script exits 1.
@@ -145,35 +196,23 @@ export async function run(options: RunOptions): Promise<RunSummary> {
   }
   // Install user plugins as additional bus subscribers BEFORE the run
   // starts emitting events. `installPlugins` runs each plugin's optional
-  // `setup` hook (the old observe-only path); `subscribeEventSinks` wires
-  // each plugin's `eventSink` capability onto the bus via wireForwarder.
-  // Both fail-fast on a setup() throw with a clean UserError naming the
-  // plugin; eventSink init failures are isolated (observability never
-  // breaks a run). With no plugin, both are no-ops.
+  // `setup` hook and fails fast on a throw with a clean UserError naming
+  // the plugin. A plugin without `setup` (the local executor and cache)
+  // subscribes nothing.
   let disposePlugins: (() => void) | undefined
-  let eventSinks: SubscribedEventSinks | undefined
   let telemetry: TelemetryHandle | undefined
-  if (prepared.workspaceConfig?.plugins && prepared.workspaceConfig.plugins.length > 0) {
-    const plugins = prepared.workspaceConfig.plugins as readonly VxPlugin[]
-    const pluginCtx = {
+  try {
+    disposePlugins = await installPlugins({
+      plugins: prepared.plugins as never,
+      bus,
       workspaceRoot: prepared.workspaceRoot,
       cacheDir: prepared.cacheDir,
-      warn: (m: string) => log.status(m),
-    }
-    try {
-      disposePlugins = await installPlugins({
-        plugins: prepared.workspaceConfig.plugins as never,
-        bus,
-        workspaceRoot: prepared.workspaceRoot,
-        cacheDir: prepared.cacheDir,
-        warn: (m) => log.status(m),
-      })
-      eventSinks = await subscribeEventSinks(plugins, bus, pluginCtx)
-    } catch (err) {
-      disposePlugins?.()
-      prepared.cache.close()
-      throw err
-    }
+      warn: (m) => log.status(m),
+    })
+  } catch (err) {
+    disposePlugins?.()
+    prepared.cache.close()
+    throw err
   }
   const {
     workspaceRoot,
@@ -192,6 +231,103 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     workspaceConfig?.concurrency ??
     Math.max(1, navigator.hardwareConcurrency)
 
+  // Resolved ONCE per run, in declaration order. A broken factory — or a
+  // workspace that declared no executor — aborts here, before any task
+  // starts.
+  let executors: readonly TaskExecutor[]
+  try {
+    executors = await resolveExecutors(prepared.plugins, {
+      workspaceRoot: prepared.workspaceRoot,
+      cacheDir: prepared.cacheDir,
+      warn: (m: string) => log.status(m),
+      concurrency,
+    })
+  } catch (err) {
+    disposePlugins?.()
+    prepared.cache.close()
+    throw err
+  }
+  // Placement: decided ONCE per task, before scheduling, so the scheduler
+  // can admit a remote-pooled task against its pool instead of a local
+  // worker slot. Group tasks run nothing; persistent tasks never reach an
+  // executor (local by construction) — both stay off the map.
+  //
+  // `--verify=inputs` pins EVERYTHING local: the input-completeness proof
+  // is the OS sandbox, which is local machinery a remote executor silently
+  // ignores — so a remotely-executed task would pass the verify VACUOUSLY,
+  // leaky or not. A verify run is a local proof procedure by definition;
+  // determinism/fingerprint modes are unaffected (no sandbox involved).
+  const placements = placeTasks(nodes, executors, options.verify?.inputs === true)
+  // A `remote: 'only'` task nobody takes succeeds WITHOUT running. That is
+  // deliberate — on a machine with no remote pool the ambient state already
+  // is what the task would have produced — but it must not be SILENT: a task
+  // reporting success while doing nothing is indistinguishable from one that
+  // worked, and the usual cause (no `cache` block, so no executor can
+  // describe its inputs to a worker) cannot be read off the outcome.
+  if (placements.remoteOnlyNoop.size > 0) {
+    const anyRemote = executors.some((e) => e.remote === true)
+    for (const id of placements.remoteOnlyNoop) {
+      const why = !anyRemote
+        ? 'no remote executor is declared'
+        : nodes.get(id)?.config.cache === undefined
+          ? 'no remote executor accepted it — the task declares no `cache` block, so there is no input set to ship'
+          : 'no remote executor accepted it'
+      log.status(`[vx] ${id}: exec.remote is 'only' and ${why} — nothing ran`)
+    }
+  }
+  // Demand: what each executor still has left to run. An executor that
+  // provisions per task (a container, an allocation, a pod) otherwise has to
+  // guess from `capacity` alone and hold everything until teardown. Built
+  // ONLY for executors that asked — with none declared there is no map, no
+  // set, and no call in the completion path.
+  const demandOf = new Map<TaskExecutor, Set<string>>()
+  for (const [id, executor] of placements.executors) {
+    if (executor.demand === undefined) continue
+    let remaining = demandOf.get(executor)
+    if (remaining === undefined) demandOf.set(executor, (remaining = new Set()))
+    remaining.add(id)
+  }
+  // The SAME set is handed back every time, narrowed in place: a consumer
+  // reading it later sees the truth rather than a stale snapshot.
+  for (const [executor, remaining] of demandOf) executor.demand!(remaining)
+  const narrowDemand =
+    demandOf.size === 0
+      ? (): void => {}
+      : (id: string): void => {
+          for (const [executor, remaining] of demandOf) {
+            if (remaining.delete(id)) executor.demand!(remaining)
+          }
+        }
+
+  // `--download` (default `all`) decides ONCE per task, here, whether a
+  // remote execution's outputs come home. `--verify` pins every task local
+  // (so nothing defers under a proof) and `all` keeps today's behaviour
+  // byte for byte.
+  // A proof must observe what it proves. `--verify=inputs` already pins
+  // every task LOCAL (so nothing could defer), but determinism and
+  // fingerprint modes do NOT — and a deferred task's outputs are absent
+  // when the verifier looks, which reported `no-outputs` for a task that
+  // declares outputs: an n/a verdict for work the proof never examined.
+  // Deferral is transfer tuning; a verify run is a rare, deliberate
+  // correctness run. Eager wins, and says so when it overrides.
+  const downloadPolicy = options.verify !== undefined ? 'all' : (options.download ?? 'all')
+  if (
+    options.verify !== undefined &&
+    options.download !== undefined &&
+    options.download !== 'all'
+  ) {
+    log.status(`vx: --verify observes outputs on disk — ignoring --download=${options.download}`)
+  }
+  const localPlaced = new Set(
+    [...nodes.keys()].filter((id) => placements.executors.get(id)?.remote !== true),
+  )
+  const download = resolveDownloadModes({
+    nodes,
+    policy: downloadPolicy,
+    localPlaced,
+    remoteOnly: placements.remoteOnly,
+  })
+
   // Run-level default task timeout (ms), applied to any task WITHOUT its own
   // `exec.timeout`. Precedence, highest first: `--timeout`/RunOptions.timeout
   // → `VX_TASK_TIMEOUT` env → workspace `timeout`. A malformed env value is
@@ -241,26 +377,68 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     process.on('SIGINT', onSigint)
     process.on('SIGTERM', onSigterm)
   }
+  // The cache handle must be released on EVERY exit path, not just the
+  // happy one: `close()` is also where the run's deferred `accessed_at`
+  // bumps are flushed, so a throw between opening the cache and the
+  // normal close leaked the SQLite handle (it matters in a long-lived
+  // host) AND lost this run's touch record, after which an LRU
+  // `vx cache prune` can evict entries the run just hit. Once-only
+  // because the normal path closes before the persistent-task wait —
+  // holding the handle open for a dev server's whole lifetime would be
+  // worse — and because close() re-runs its retention DELETEs.
+  let cacheClosed = false
+  const closeCache = (): void => {
+    if (cacheClosed) return
+    cacheClosed = true
+    cache.close()
+  }
   try {
     // One run-id per `vx run` invocation. Every task in the resulting
     // graph carries it so analytics queries can group by invocation.
     const runId = ulid()
     const runStartHrTimeNs = process.hrtime.bigint()
     const endedAtMsAtStart = Date.now()
-    const remoteCacheEnabled = cache instanceof LayeredCache
-    const policy: CachePolicy = options.cache ?? FULL_CACHE_POLICY
-
-    // `--verify` observes the miss-then-save path, so a policy with NO write
-    // axis (`--no-cache`, `--cache=local:,remote:`) verifies nothing — every
-    // task would come back green with zero verdicts. The user asked for a
-    // proof; silently skipping it is the one failure mode verification must
-    // never have (same platform-honesty rule as the sandbox-unavailable
-    // error). `--force --verify` is the supported re-verify-warm recipe.
-    if (options.verify !== undefined && !policy.localWrite && !policy.remoteWrite) {
-      prepared.cache.close()
+    const remoteCacheEnabled = prepared.hasRemoteLayer
+    // Normalised ONCE so every consumer — the verify gate, execute-task,
+    // the dedup predicate, the recorded invocation row — reads the policy
+    // that actually governed the run. Reading the raw request here made
+    // tasks clean their outputs before every exec for a save that never
+    // happened, and `--verify` clean them AGAIN and restore an artifact
+    // that was never written (wiping a successful build's tree and
+    // reporting it failed).
+    const policy: CachePolicy = effectiveCachePolicy(
+      options.cache ?? FULL_CACHE_POLICY,
+      prepared.hasRemoteLayer,
+    )
+    const deferredOutputs = new DeferredOutputs({
+      nodes,
+      cache,
+      workspaceRoot,
+      nestedDirsByProject,
+      ...(gitFilesCache !== undefined ? { gitFilesCache } : {}),
+      localWrite: policy.localWrite,
+    })
+
+    // `--verify` observes the miss-then-save path and then RESTORES attempt
+    // 1 from the artifact that save wrote, so the tree ends byte-identical
+    // to what was cached regardless of the verdict. That restore reads the
+    // LOCAL artifact file — `restoreOutputs` is a local extraction on every
+    // layer, by design — so without the local WRITE axis there is nothing to
+    // restore from: verify cleaned a successful build's declared outputs and
+    // then failed the run on the missing artifact. A remote fallback would
+    // not fix that, only narrow it to "whenever the remote is reachable at
+    // that instant", turning a guarantee into a coin flip with data loss on
+    // the losing side. So the honest gate is the local write axis, and
+    // `--no-cache` / `--cache=local:,…` / `--cache=local:r,…` are refused
+    // loudly — silently verifying nothing is the one failure mode
+    // verification must never have (same platform-honesty rule as the
+    // sandbox-unavailable error). `--force --verify` re-verifies a warm graph.
+    if (options.verify !== undefined && !policy.localWrite) {
       throw new UserError(
-        '--verify needs cache writes to prove anything (it verifies the save path); ' +
-          'drop --no-cache, or use --force --verify to re-execute and verify a warm graph',
+        '--verify needs the LOCAL cache write axis: it re-runs the task, then restores ' +
+          'attempt 1 from the local artifact so the outputs on disk are the ones that were ' +
+          'cached. Enable local writes (e.g. --cache=local:w,remote:rw), drop --no-cache, ' +
+          'or use --force --verify to re-execute and verify a warm graph',
       )
     }
 
@@ -275,16 +453,21 @@ export async function run(options: RunOptions): Promise<RunSummary> {
 
     // The canonical run-context record — the same git/CI/host data the
     // invocation header uses, shaped as the telemetry export contract.
-    // Built + consulted ONLY when plugins are declared (an explicit opt-in),
-    // and BEFORE run:start is emitted so a sink catches the whole stream.
+    // Built + consulted ONLY when a plugin CONTRIBUTES `telemetry`, and
+    // BEFORE run:start is emitted so a sink
+    // catches the whole stream.
     // subscribeTelemetry returns undefined when no sink is contributed (no
     // telemetry plugin, or all declined — e.g. otel() with no OTLP endpoint),
     // so a plain run does ZERO extra work: no record allocation, no bus
     // subscriber, no summary building. The hot path stays off-limits.
     let runContextRecord: RunContextRecord | undefined
-    const hasPlugins =
-      prepared.workspaceConfig?.plugins !== undefined && prepared.workspaceConfig.plugins.length > 0
-    if (hasPlugins || options.telemetrySinks !== undefined) {
+    // Gate on a plugin that can actually CONTRIBUTE a sink, not on declaring
+    // any plugin at all: a cache-only plugin has no telemetry hook to
+    // consult, so paying for the record (2 git spawns + a `.vx/workspace-id`
+    // write on a remote-less repo) buys nothing. A plugin that has the hook
+    // but declines still pays — its answer is only knowable by asking.
+    const hasTelemetryPlugin = prepared.plugins.some((p) => p.telemetry !== undefined)
+    if (hasTelemetryPlugin || options.telemetrySinks !== undefined) {
       // Workspace identity (telemetry v2): one git spawn, paid only when a
       // telemetry consumer can exist — a plain run never reaches here.
       const wsIdentity = captureWorkspaceIdentity(workspaceRoot)
@@ -298,6 +481,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
         flow: options.flow ?? null,
         commitSha: gitContext.commitSha,
         branch: gitContext.branch,
+        defaultBranch: captureDefaultBranch(process.env, workspaceRoot),
         dirty: gitContext.dirty,
         ci: ciContext.ci,
         ciProvider: ciContext.provider,
@@ -309,7 +493,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
         tags: options.tags ?? {},
       }
       telemetry = await subscribeTelemetry(
-        (prepared.workspaceConfig?.plugins ?? []) as readonly VxPlugin[],
+        prepared.plugins,
         bus,
         { workspaceRoot, cacheDir, warn: (m: string) => log.status(m) },
         runContextRecord,
@@ -329,7 +513,6 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     if (anySandboxed) {
       const avail = await probeSandbox()
       if (!avail.available) {
-        prepared.cache.close()
         throw new UserError(
           verifyInputs
             ? `--verify=inputs needs the sandbox, which is not available: ${avail.reason}`
@@ -370,7 +553,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     // Nothing declared → empty map → fields omitted from the scheduler
     // AND the footer → byte-identical legacy path.
     const memBudget = options.memory ?? os.totalmem()
-    const resourceCosts = resolveResourceCosts(nodes, concurrency, memBudget)
+    const resourceCosts = resolveResourceCosts(nodes)
 
     // Run context for the footer. The top-of-run header is gone — the
     // banner now lives in the summary, where the eye lands at the end.
@@ -385,17 +568,23 @@ export async function run(options: RunOptions): Promise<RunSummary> {
 
     // Lifecycle hooks drive the default logger's dynamic status line
     // (TTY-only); custom loggers may ignore them.
-    log.runStart?.({ total: taskCount, concurrency, requestedCount, context: runContext })
+    log.runStart?.({
+      total: taskCount,
+      concurrency,
+      requestedCount,
+      context: runContext,
+      startedAtMs: endedAtMsAtStart,
+    })
 
     // Remote-only: kick off background prefetches so remote-GET latency
     // overlaps execution. Fire-and-forget — execution starts on the next
-    // line; LayeredCache ingests hits into local and de-dups so
+    // line; the layer ingests hits into local and de-dups so
     // execute-task's cache.get awaits the in-flight promise (one remote
     // GET per key). Gated entirely on a remote layer being configured;
     // local-only runs never reach here, so their behavior + perf is
     // unchanged (no upfront key pass, no local probing).
     let prefetchDone: Promise<void> = Promise.resolve()
-    if (cache instanceof LayeredCache) {
+    if (prepared.hasRemoteLayer) {
       prefetchDone = startRemotePrefetch({
         nodes,
         cache,
@@ -416,8 +605,8 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     // restore-tier the scheduler runs ahead of their deps but only as
     // worker-slot backfill (misses own the pool) — and execute reuses
     // each probe, so there is no second cache.get. Gated by
-    // shouldShortCircuit (localRead + has-deps); when off, both maps are
-    // empty and the run is byte-identical.
+    // shouldShortCircuit (local reads on, no remote layer); when off, both
+    // maps are empty and the run is byte-identical.
     let shortCircuit: ShortCircuit = EMPTY_SHORT_CIRCUIT
     if (shouldShortCircuit(nodes, policy, cache)) {
       shortCircuit = await startLocalShortCircuit({
@@ -447,6 +636,11 @@ export async function run(options: RunOptions): Promise<RunSummary> {
         ...(taskTimeoutDefault !== undefined ? { timeout: taskTimeoutDefault } : {}),
         ...(options.verify !== undefined ? { verify: options.verify } : {}),
         log,
+        executor: placements.executors.get(node.id) ?? UNPLACED_EXECUTOR,
+        ...(download.modeOf.get(node.id) === 'deferred' ? { download: 'deferred' as const } : {}),
+        deferred: deferredOutputs,
+        ...(placements.remoteOnlyNoop.has(node.id) ? { remoteOnlyNoop: true } : {}),
+        ...(placements.remoteOnly.has(node.id) ? { remoteOnly: true } : {}),
         nestedProjectDirs: nestedDirsByProject.get(node.projectName) ?? [],
         runStartHrTimeNs,
         persistentRegistry,
@@ -526,9 +720,11 @@ export async function run(options: RunOptions): Promise<RunSummary> {
       }
     }
 
+    mark('classify + probe')
     const outcomes = await runGraph({
       nodes,
       concurrency,
+      ...(hasPooledExecutor(executors) ? { poolOf: poolOfPlacement(placements) } : {}),
       ...(resourceCosts.size > 0 ? { resourceCosts, cpuBudget: concurrency, memBudget } : {}),
       ...(options.continueMode !== undefined ? { continueMode: options.continueMode } : {}),
       onStart: (node) => {
@@ -536,11 +732,12 @@ export async function run(options: RunOptions): Promise<RunSummary> {
       },
       onFinish: (o) => {
         log.taskComplete(o.node, o)
+        narrowDemand(o.node.id)
       },
       execute: executeWithDedup,
-      // Predictive scheduling: empty map when not opted in, in which
-      // case the scheduler keeps the static baseline behavior.
-      priorities: prepared.priorities,
+      // A `schedule` plugin's weights; the scheduler keeps its structural
+      // baseline as the tie-break. Empty map → baseline only.
+      ...(prepared.priorities.size > 0 ? { priorities: prepared.priorities } : {}),
       // Local short-circuit: confirmed stable local hits the scheduler
       // runs ahead of their deps as low-priority worker-slot backfill.
       // Empty when the short-circuit didn't fire → byte-identical.
@@ -571,27 +768,39 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     }
     const keepAliveSet = new Set(keepAlive)
 
-    // Shut down the dependency-only persistent tasks before reporting the
-    // final summary. SIGTERM gives well-behaved servers (vite, next,
-    // esbuild --watch) a moment to clean up; we don't escalate to SIGKILL
-    // — process-group propagation on Ctrl-C handles the unhappy case.
-    // Bun's Subprocess.kill is idempotent on an already-exited child.
-    for (const child of persistentRegistry.values()) {
-      if (!keepAliveSet.has(child)) child.kill('SIGTERM')
+    // Shut down the dependency-only persistent tasks before reporting the final
+    // summary. SIGTERM gives well-behaved servers (vite, next, esbuild --watch)
+    // a moment to clean up. Bun's Subprocess.kill is idempotent on an
+    // already-exited child. Bound the wait: a persistent dep that traps or
+    // ignores SIGTERM (a wedged mock server) would otherwise hang the run at
+    // NORMAL completion forever — after a grace, SIGKILL the stragglers and move
+    // on. Well-behaved servers exit in well under the grace, so the happy path
+    // pays nothing; the timer is cleared + unref'd so a fast shutdown never
+    // delays CLI exit.
+    const dyingChildren = [...persistentRegistry.values()].filter((c) => !keepAliveSet.has(c))
+    for (const child of dyingChildren) child.kill('SIGTERM')
+    const allExited = Promise.allSettled(dyingChildren.map((c) => c.exited))
+    let graceTimer: ReturnType<typeof setTimeout> | undefined
+    const winner = await Promise.race([
+      allExited.then(() => 'exited' as const),
+      new Promise<'grace'>((resolve) => {
+        graceTimer = setTimeout(() => resolve('grace'), PERSISTENT_SHUTDOWN_GRACE_MS)
+        graceTimer.unref?.()
+      }),
+    ])
+    if (graceTimer !== undefined) clearTimeout(graceTimer)
+    if (winner === 'grace') {
+      for (const child of dyingChildren) child.kill('SIGKILL')
+      await allExited
     }
-    await Promise.allSettled(
-      [...persistentRegistry.values()].filter((c) => !keepAliveSet.has(c)).map((c) => c.exited),
-    )
 
+    mark('run graph')
     // Clear the status line for good before the summary prints.
     log.runEnd?.()
 
     const list = [...outcomes.values()]
     const ok =
-      list.every(
-        (o) =>
-          o.status === 'success' || o.status === 'cache-hit' || o.status === 'cache-hit-remote',
-      ) &&
+      list.every((o) => isPassStatus(o.status)) &&
       // `--verify`: a provably-unsafe cache entry (non-deterministic outputs, a
       // re-run that failed, or a read of undeclared inputs) turns the run red so
       // CI catches it.
@@ -613,6 +822,19 @@ export async function run(options: RunOptions): Promise<RunSummary> {
       for (const line of formatPersistentList(keepAliveNodes, colors)) log.status(line)
     }
     for (const line of formatRunSummary(list, totalMs, colors, runContext)) log.status(line)
+    // A task killed by a shutdown signal is in no bucket above, yet it makes
+    // `ok` false — name it, or the red exit is undiagnosable.
+    for (const line of formatAbortedSection(list)) log.status(line)
+    // Outputs that never came home are not an error, but a silent `dist/`
+    // that is empty-or-stale would be: name every task whose bytes are
+    // still remote.
+    const stillDeferred = deferredOutputs.pending()
+    if (stillDeferred.length > 0) {
+      log.status('')
+      log.status(
+        `  Deferred: ${stillDeferred.length} task(s) left outputs remote (--download=none): ${stillDeferred.join(', ')}`,
+      )
+    }
     if (options.verify !== undefined) {
       for (const line of formatVerifySection(list)) log.status(line)
       // A fingerprint-only run attaches no verdicts, so the verdict-driven
@@ -644,6 +866,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
           startedAtMs: endedAtMsAtStart,
           endedAtMs,
           totalMs,
+          ok,
           outcomes: list,
         })
         log.status(`vx: summary written to ${wrote}`)
@@ -683,7 +906,6 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     let hitLocalCount = 0
     let hitRemoteCount = 0
     for (const o of list) {
-      if (!o.hash) continue
       if (isGroupTask(o.node)) continue
       // aborted (killed by a shutdown signal) isn't a real run.
       if (o.status === 'aborted') continue
@@ -700,6 +922,8 @@ export async function run(options: RunOptions): Promise<RunSummary> {
         if (o.hash !== undefined) t.hash = o.hash
         if (o.cpuMs !== undefined) t.cpuMs = o.cpuMs
         if (o.peakRssBytes !== undefined) t.peakRssBytes = o.peakRssBytes
+        if (o.where !== undefined) t.where = o.where
+        if (o.outputs !== undefined) t.outputs = o.outputs
         if (o.attempts !== undefined) t.attempts = o.attempts
         if (o.verify !== undefined) t.verify = o.verify
         if (o.outputFp !== undefined) t.outputFp = o.outputFp
@@ -707,8 +931,17 @@ export async function run(options: RunOptions): Promise<RunSummary> {
         if (o.wallclockEndNs !== undefined) t.wallclockEndNs = o.wallclockEndNs.toString()
         summaryTasks.push(t)
       }
+      // Every non-group, non-aborted outcome gets a row — the same set
+      // `tallyOutcomes` counts, so `invocations.task_count` equals both the
+      // terminal's "N total" and `COUNT(*) FROM runs WHERE run_id = ?`.
+      // An outcome with NO hash (a `skipped` task never probed the cache; a
+      // `persistent` one is never cacheable) used to be dropped here because
+      // `runs.hash` is NOT NULL — which made a failing persistent task record
+      // `0 tasks, 0 failures` on a run the terminal called red, and a failed
+      // task with a skipped dependent record 1 of 2. `bindRun` stores `''` for
+      // those instead; the key-diff readers guard it.
       toRecord.push({
-        hash: o.hash,
+        ...(o.hash !== undefined ? { hash: o.hash } : {}),
         project: o.node.projectName,
         task: o.node.taskName,
         status: o.status,
@@ -717,9 +950,11 @@ export async function run(options: RunOptions): Promise<RunSummary> {
         ...(options.forwardArgs !== undefined ? { forwardArgs: options.forwardArgs } : {}),
         // Anchor to the REAL per-task wall-clock window: run-start wall time +
         // the task's ns offset (captured for hits and executed tasks alike).
-        // The `now - duration` fallback only applies to outcomes without an
-        // offset (none today). Using run-end-minus-duration for everything was
-        // the old bug that piled every task at the right edge of the timeline.
+        // The `now - duration` fallback applies to outcomes without an offset —
+        // today only `skipped`, which the scheduler finishes synchronously with
+        // no span, so it collapses to a zero-width mark at the run's end. Using
+        // run-end-minus-duration for EVERYTHING was the old bug that piled every
+        // task at the right edge of the timeline.
         startedAt:
           o.wallclockStartNs !== undefined
             ? endedAtMsAtStart + Math.round(Number(o.wallclockStartNs) / 1e6)
@@ -733,7 +968,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
         ...(o.peakRssBytes !== undefined ? { peakRssBytes: o.peakRssBytes } : {}),
         ...(o.wallclockStartNs !== undefined ? { wallclockStartNs: o.wallclockStartNs } : {}),
         ...(o.wallclockEndNs !== undefined ? { wallclockEndNs: o.wallclockEndNs } : {}),
-        cacheHit: o.status === 'cache-hit' || o.status === 'cache-hit-remote',
+        cacheHit: isCacheHit(o.status),
         ...(o.attempts !== undefined ? { attempts: o.attempts } : {}),
       })
       if (o.status === 'failed') failedCount++
@@ -768,41 +1003,26 @@ export async function run(options: RunOptions): Promise<RunSummary> {
       tags: JSON.stringify(options.tags ?? {}),
     }
     cache.recordRunBundle({ runs: toRecord, invocation })
+    mark('record history')
     // Hand the per-run summary to the telemetry sinks + drain them. Only
     // when a sink is active (telemetry !== undefined) — otherwise this
     // whole block is skipped and the run is byte-identical to before.
     // emitSummary/flush are crash-isolated, so a faulty sink can't fail
     // the run; flush is the sink's last chance to ship buffered records.
     if (telemetry !== undefined && runContextRecord !== undefined) {
-      const summary: RunSummaryRecord = {
-        v: TELEMETRY_SCHEMA_VERSION,
-        run: runContextRecord,
+      const summary = assembleRunSummary(runContextRecord, summaryTasks, {
         startedAt: endedAtMsAtStart,
         endedAt: endedAtMs,
         totalDurationMs: Math.round(totalMs),
-        taskCount: toRecord.length,
-        failedCount,
-        hitCount: hitLocalCount + hitRemoteCount,
-        hitLocalCount,
-        hitRemoteCount,
         exitOk: ok,
-        tasks: summaryTasks,
-      }
+      })
       telemetry.emitSummary(summary)
       await telemetry.flush()
     }
-    // End-of-run plugin lifecycle: each event sink's flush() (its last
-    // chance to ship buffered records) and each plugin's teardown().
-    // Crash-isolated + time-bounded inside teardownPlugins, so a faulty
-    // plugin can neither fail nor hang the run. Normal completion path
-    // only — the finally below just unsubscribes.
-    if (prepared.workspaceConfig?.plugins && prepared.workspaceConfig.plugins.length > 0) {
-      await teardownPlugins(
-        prepared.workspaceConfig.plugins as readonly VxPlugin[],
-        eventSinks?.sinks ?? [],
-        (m) => log.status(m),
-      )
-    }
+    // End-of-run plugin lifecycle: each plugin's teardown(). Crash-isolated
+    // + time-bounded inside teardownPlugins, so a faulty plugin can neither
+    // fail nor hang the run. Normal completion path only — the finally
+    // below just unsubscribes.
     // Drain any still-in-flight background prefetches before closing the
     // cache handle — a prefetch ingesting into a closed SQLite DB would
     // throw. Tasks that resolved as local hits never awaited their
@@ -811,9 +1031,19 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     // before scheduling, so nothing of its is in flight here.) The
     // background write-through uploads queued by LayeredCache.save
     // settle here for the same reason.
+    //
+    // BEFORE teardownPlugins, and that order is load-bearing: the drain
+    // pushes bytes through a layer a PLUGIN provided, so a plugin that
+    // releases its client in `teardown()` would otherwise have the
+    // channel shut from under the upload — silently losing every remote
+    // write with nothing but a warning. The seam's contract is that a
+    // cache layer stays usable until the run's uploads have settled.
     await prefetchDone
-    if (cache instanceof LayeredCache) await cache.drainUploads()
-    cache.close()
+    await cache.drainUploads?.()
+    await teardownPlugins(prepared.plugins, (m) => log.status(m))
+    closeCache()
+    mark('close')
+    printTimings()
 
     // Tear down SRT's network bridge + (on macOS) log monitor. No-op if
     // no task was sandboxed; otherwise SRT keeps proxy servers alive and
@@ -849,8 +1079,17 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     // Plugins installed at the top of run() get their bus subscriptions
     // released here. Idempotent; safe even if installPlugins threw.
     disposePlugins?.()
-    eventSinks?.dispose()
     telemetry?.dispose()
+    // No-op after the normal path's close. On a throw this is the only
+    // close there is, and it must not itself throw — that would replace
+    // the run's real error with a teardown one. Background uploads are
+    // NOT drained here: they hold no database state, and awaiting a
+    // wedged remote would turn a failing run into a hanging one.
+    try {
+      closeCache()
+    } catch {
+      // teardown must not throw on the way out
+    }
   }
 }
 
@@ -869,19 +1108,248 @@ export async function planRun(options: RunOptions): Promise<RunPlan> {
   const log = options.log ?? defaultLogger()
   const prepared = await prepareRun(options, log)
   try {
+    if (prepared.unresolvedTasks.length > 0) {
+      return { tasks: [], unresolvedTasks: prepared.unresolvedTasks }
+    }
     if (prepared.empty !== null) return { tasks: [] }
     return await plan({
       nodes: prepared.nodes,
       workspaceRoot: prepared.workspaceRoot,
       workspaceFingerprint: prepared.workspaceFingerprint,
       cache: prepared.cache,
-      cachePolicy: options.cache ?? FULL_CACHE_POLICY,
+      cachePolicy: effectiveCachePolicy(
+        options.cache ?? FULL_CACHE_POLICY,
+        prepared.hasRemoteLayer,
+      ),
       forwardArgs: options.forwardArgs,
       nestedDirsByProject: prepared.nestedDirsByProject,
       gitFilesCache: prepared.gitFilesCache,
       hashCache: prepared.hashCache,
+      // Plan-time duration prediction (dev-scenarios S2): the same local
+      // history the predictive scheduler reads, folded into the plan as
+      // per-task p50s + a predicted wall-clock. `--dry` is an explicit
+      // inspection command, so the history read's cost is fine here even
+      // though it stays OFF the default run path.
+      history: new LocalHistoryProvider(prepared.localCache.dbHandle()),
+      // Placement, by the same rules the run applies. Only worth showing
+      // when there is a choice to show — with one declared executor every
+      // line would carry the same label. Resolving the executors here is
+      // the same plugin-factory call `prepareRun` already makes for the
+      // cache capability, so plan mode gains no new class of side effect.
+      ...(await planExecutorOf(prepared, log, options.download ?? 'all')),
     })
   } finally {
     prepared.cache.close()
   }
 }
+
+/**
+ * A task is pinned to this machine when it is persistent, transitively
+ * depends on a persistent task (a worker cannot reach a port on the
+ * submitter), or declares `exec.remote: false`.
+ */
+function pinnedLocalSet(nodes: Map<string, TaskNode>): Set<string> {
+  const pinned = new Set<string>()
+  const memo = new Map<string, boolean>()
+  const visit = (id: string): boolean => {
+    const known = memo.get(id)
+    if (known !== undefined) return known
+    const node = nodes.get(id)
+    if (node === undefined) return false
+    memo.set(id, false) // cycle guard; the graph builder already rejects cycles
+    const result =
+      node.config.exec?.persistent !== undefined ||
+      node.config.exec?.remote === false ||
+      node.deps.some((d) => visit(d))
+    memo.set(id, result)
+    if (result) pinned.add(id)
+    return result
+  }
+  for (const id of nodes.keys()) visit(id)
+  return pinned
+}
+
+interface Placements {
+  executors: Map<string, TaskExecutor>
+  /**
+   * `exec.remote: 'only'` tasks that no REMOTE executor took — a NO-OP on
+   * this machine: never executed, declared outputs never cleaned or
+   * restored. The task exists to produce a remote input tree; without a
+   * remote pool, dependents use the machine's ambient state exactly as they
+   * did before the field existed.
+   */
+  remoteOnlyNoop: Set<string>
+  /** `'only'` tasks a remote executor DID take — executed remotely, outputs stay remote. */
+  remoteOnly: Set<string>
+}
+
+function placeTasks(
+  nodes: Map<string, TaskNode>,
+  executors: readonly TaskExecutor[],
+  pinAllLocal = false,
+): Placements {
+  const pinned = pinnedLocalSet(nodes)
+  const placements: Placements = {
+    executors: new Map(),
+    remoteOnlyNoop: new Set(),
+    remoteOnly: new Set(),
+  }
+  for (const node of nodes.values()) {
+    if (isGroupTask(node) || node.config.exec?.persistent !== undefined) continue
+    const executor = selectExecutor(executors, {
+      taskId: node.id,
+      projectName: node.projectName,
+      projectDir: node.projectDir,
+      command: node.config.exec!.command,
+      pinnedLocal: pinAllLocal || pinned.has(node.id),
+      cacheable: node.config.cache !== undefined,
+      ...(node.config.exec?.resources === undefined
+        ? {}
+        : { resources: node.config.exec.resources }),
+    })
+    placements.executors.set(node.id, executor)
+    if (node.config.exec?.remote === 'only') {
+      // A pinned 'only' task (it transitively depends on a persistent one)
+      // lands here too: pinning wins, so it noops rather than shipping.
+      if (executor.remote === true) placements.remoteOnly.add(node.id)
+      else placements.remoteOnlyNoop.add(node.id)
+    }
+  }
+  return placements
+}
+
+/**
+ * `executorOf` for `planRun`, or nothing. Declining plugins, a single
+ * executor, or a resolution error all yield nothing: `--dry` is an
+ * inspection command and must not fail over a label.
+ */
+async function planExecutorOf(
+  prepared: Awaited<ReturnType<typeof prepareRun>>,
+  log: Logger,
+  policy: 'all' | 'toplevel' | 'none',
+): Promise<{
+  executorOf?: (id: string) => string | undefined
+  downloadOf?: (id: string) => 'eager' | 'deferred' | 'never' | undefined
+  downloadDowngrades?: ReadonlyArray<{ taskId: string; reason: string }>
+}> {
+  let executors: readonly TaskExecutor[]
+  try {
+    executors = await resolveExecutors(prepared.plugins, {
+      workspaceRoot: prepared.workspaceRoot,
+      cacheDir: prepared.cacheDir,
+      warn: (m: string) => log.status(m),
+      concurrency: Math.max(1, navigator.hardwareConcurrency),
+    })
+  } catch {
+    return {}
+  }
+  const placements = placeTasks(prepared.nodes, executors)
+  // Download modes need placement regardless of how many executors there
+  // are (a single REMOTE one still defers); executor LABELS only earn their
+  // column when there is a choice to report.
+  const download =
+    policy === 'all'
+      ? undefined
+      : resolveDownloadModes({
+          nodes: prepared.nodes,
+          policy,
+          localPlaced: new Set(
+            [...prepared.nodes.keys()].filter(
+              (id) => placements.executors.get(id)?.remote !== true,
+            ),
+          ),
+          remoteOnly: placements.remoteOnly,
+        })
+  return {
+    ...(executors.length < 2
+      ? {}
+      : {
+          executorOf: (id: string) =>
+            placements.remoteOnlyNoop.has(id) ? 'noop' : placements.executors.get(id)?.name,
+        }),
+    ...(download === undefined
+      ? {}
+      : {
+          downloadOf: (id: string) => download.modeOf.get(id),
+          downloadDowngrades: [...download.downgrades].map(([taskId, reason]) => ({
+            taskId,
+            reason,
+          })),
+        }),
+  }
+}
+
+/**
+ * Stands in for the executor of a task that was never placed — a group task
+ * (runs nothing) or a persistent one (`executePersistentTask` owns it and
+ * never reads this field). It THROWS rather than silently picking some
+ * executor from the list, so a refactor that routes such a task through the
+ * exec path fails loudly instead of shipping a localhost server to a worker.
+ */
+const UNPLACED_EXECUTOR: TaskExecutor = {
+  name: 'unplaced',
+  execute: (req) => {
+    throw new Error(`internal error: ${req.taskId} reached an executor without being placed`)
+  },
+}
+
+function hasPooledExecutor(executors: readonly TaskExecutor[]): boolean {
+  return executors.some((e) => e.capacity !== undefined)
+}
+
+function poolOfPlacement(
+  placements: Placements,
+): (id: string) => { name: string; capacity: number } | undefined {
+  return (id) => {
+    const executor = placements.executors.get(id)
+    return executor?.capacity === undefined
+      ? undefined
+      : { name: executor.name, capacity: executor.capacity }
+  }
+}
+
+/**
+ * A typo's nearest declared name, when one is within two edits — the
+ * message names the fix instead of only the mistake. A bare name is
+ * matched against every declared task; `pkg#task` against the project
+ * names first (the task kept) and then against that project's tasks, so
+ * the hint is a spec the user can run.
+ */
+function didYouMean(
+  unresolved: readonly string[],
+  projects: ReadonlyMap<string, ProjectEntry>,
+): string {
+  const nearest = (name: string, candidates: Iterable<string>): string | undefined => {
+    let best: string | undefined
+    let bestD = 3
+    for (const c of candidates) {
+      const d = editDistance(name, c)
+      if (d < bestD) {
+        bestD = d
+        best = c
+      }
+    }
+    return best === name ? undefined : best
+  }
+  const tasksOf = (p: ProjectEntry | undefined): string[] => Object.keys(p?.config.tasks ?? {})
+  const allTasks = new Set<string>()
+  for (const p of projects.values()) for (const t of tasksOf(p)) allTasks.add(t)
+  const hints: string[] = []
+  for (const spec of unresolved) {
+    const at = spec.indexOf('#')
+    if (at < 0) {
+      const t = nearest(spec, allTasks)
+      if (t !== undefined) hints.push(t)
+      continue
+    }
+    const [proj, task] = [spec.slice(0, at), spec.slice(at + 1)]
+    if (!projects.has(proj)) {
+      const p = nearest(proj, projects.keys())
+      if (p !== undefined && tasksOf(projects.get(p)).includes(task)) hints.push(`${p}#${task}`)
+      continue
+    }
+    const t = nearest(task, tasksOf(projects.get(proj)))
+    if (t !== undefined) hints.push(`${proj}#${t}`)
+  }
+  return hints.length === 0 ? '' : ` Did you mean ${hints.join(', ')}?`
+}