@vzn/vx 0.0.18 → 0.0.22

package/src/orchestrator/options.ts

@@ -8,6 +8,7 @@ import type { TaskOutcome } from '../graph/index.js'
 import type { EventBus } from './events.js'
 import type { TelemetrySink } from './telemetry.js'
 import type { Logger } from './logger.js'
+import type { ProjectEntry } from '../workspace/index.js'
 
 export interface RunOptions {
   cwd: string
@@ -19,6 +20,16 @@ export interface RunOptions {
    */
   tasks: readonly string[]
   projects?: string[]
+  /**
+   * Configs a selection pass in this same process already loaded and
+   * staged with the same `cacheDir` and `frozen` (the CLI walks every
+   * `pkg#task` edge for `--filter 'app...'` and `--affected`). The run
+   * seeds and scopes as it always does and reuses these entries instead
+   * of evaluating and staging them again. One run only — never carry a
+   * map across runs, a config can change between them (`vx watch` drops
+   * it per cycle).
+   */
+  staged?: ReadonlyMap<string, ProjectEntry>
   concurrency?: number
   /**
    * Cache directory override (`--cache-dir <path>`). Absolute, or relative
@@ -34,6 +45,13 @@ export interface RunOptions {
    * both reads false (re-execute, still refresh the cache).
    */
   cache?: CachePolicy
+  /**
+   * The caller asked for a remote axis by name (`--cache=remote:rw`). With
+   * no cache plugin supplying a remote layer the axes are inert, and a CI
+   * job that believes it is filling a shared cache deserves one line
+   * saying it is not.
+   */
+  remoteRequested?: boolean
   /**
    * CI mode: load configs FROM the committed vx-lock.json instead of
    * evaluating them (frozen-env reproducibility). Requires the lock
@@ -62,7 +80,7 @@ export interface RunOptions {
    * task in the run actually needs them (Bazel's "build without the
    * bytes").
    * Pure transfer tuning: it can never change what a command produces,
-   * so it is NEVER folded into a cache key (a RunOption, like --verify).
+   * so it is NEVER folded into a cache key: it is a RunOption, not config.
    * Locally-placed tasks always write in place and ignore this.
    */
   download?: 'all' | 'toplevel' | 'none'
@@ -82,24 +100,6 @@ export interface RunOptions {
    * the same run with and without `--retry` derives identical keys.
    */
   retries?: number
-  /**
-   * Cache-correctness verification (`vx run --verify`). When set, an
-   * executed + cacheable task is re-run after its save and its outputs are
-   * content-compared: a divergence means the task is non-hermetic and its
-   * cache entry is provably unsafe (see docs/design/cache-correctness).
-   * `fingerprint` computes + ships an output-tree fingerprint on executed
-   * tasks (no re-run — the cross-machine diff feed; `--verify`/`=all` set
-   * it too, for free). `allow` (from `--verify-allow=<pkg#task>,…`) exempts
-   * known-nondeterministic tasks from failing the run. A pure side-channel —
-   * never folded into a cache key; the re-run never saves. Undefined = off
-   * (a plain run is byte-identical).
-   */
-  verify?: {
-    determinism: boolean
-    inputs: boolean
-    fingerprint: boolean
-    allow: ReadonlySet<string>
-  }
   /**
    * Run-level default task timeout (ms), for tasks that declare no
    * `exec.timeout`. Highest of the run-level defaults: `--timeout` /
@@ -109,14 +109,6 @@ export interface RunOptions {
    * cached), so a `--timeout` run cache-hits a plain run's entry.
    */
   timeout?: number
-  /**
-   * Memory budget (bytes) that per-task `exec.resources.memory`
-   * reservations pack against (`--memory <size>`). Defaults to
-   * `os.totalmem()` — override it in cgroup-limited containers, where
-   * totalmem() reports the HOST's RAM, not the limit. A per-run
-   * scheduling knob only — never folded into any cache key.
-   */
-  memory?: number
   /**
    * Failure propagation: 'never' stops dispatch on the first failure
    * (in-flight tasks finish; everything queued skips), 'deps-ok'
@@ -147,13 +139,33 @@ export interface RunOptions {
    */
   profile?: string
   /**
-   * Install SIGINT/SIGTERM handlers for the run's duration: forward
-   * SIGTERM to every live child, close the cache, exit 128+signo
-   * (130/143). Default true. The watch loop disables this — it owns
+   * Install SIGINT/SIGTERM handlers for the run's duration: forward the
+   * signal received to every live child's group (a SIGHUP as SIGTERM),
+   * close the cache, exit 128+signo (130/143). Default true. The watch loop disables this — it owns
    * signal disposition for its whole lifetime and a cycle must never
    * exit the process out from under it.
    */
   handleSignals?: boolean
+  /**
+   * Abort the run from outside. On `abort` every live and persistent
+   * child is sent SIGINT when the abort reason is `'SIGINT'` (a Ctrl-C
+   * the embedder received), SIGTERM otherwise, given the kill grace,
+   * then SIGKILLed; nothing
+   * further is dispatched; run() returns with the killed tasks and the
+   * never-started ones `aborted`. The seam for an embedder that owns its
+   * process's lifetime instead of `handleSignals` — the watch loop aborts
+   * its in-flight cycle on SIGINT/SIGTERM this way, so a cancellation
+   * never orphans a cycle's children.
+   */
+  signal?: AbortSignal
+  /**
+   * Hand the requested persistent tasks still running back on
+   * `RunSummary.persistent` instead of stopping them when the graph ends.
+   * The caller owns them from then on and stops them with its `stop()`.
+   * The watch loop sets this: a dev server stays up while watch idles and
+   * is replaced only when the next cycle starts.
+   */
+  holdPersistent?: boolean
   log?: Logger
   /**
    * Inject the run's event bus. When provided, the orchestrator emits
@@ -168,7 +180,8 @@ export interface RunOptions {
   bus?: EventBus
   /**
    * Shared in-flight execution registry, keyed by task hash. Supplied by a
-   * long-lived service (`vx serve`) so concurrent runs DEDUP work: a task
+   * long-lived embedder (a daemon built on the façade; core ships none) so
+   * concurrent runs DEDUP work: a task
    * already executing for one run is awaited by another (which then
    * restores the just-saved artifact from cache) instead of re-running.
    * A stateless `vx run` passes none and is byte-identical to before.
@@ -183,8 +196,8 @@ export interface RunOptions {
   /**
    * Additional observe-only telemetry sinks, merged with whatever the
    * workspace plugins contribute. The embedder seam: a host executing
-   * runs on behalf of others (a serve recording delegated runs
-   * into its ingest store) attaches a sink without needing a workspace
+   * runs on behalf of others (a daemon recording delegated runs
+   * into its own store) attaches a sink without needing a workspace
    * plugin. Undefined → zero cost, identical to before; the telemetry
    * host's zero-sink invariant still applies when both sources are
    * empty.
@@ -200,16 +213,32 @@ export interface RunOptions {
   /**
    * An injected remote cache layer — the embedder seam mirroring
    * `telemetrySinks`: a host that already holds a wire client (a
-   * distribution agent, a serve executing on behalf of a submitter)
+   * distribution agent, a daemon executing on behalf of a submitter)
    * passes it here and run() composes `LayeredCache(local, injected)`.
    * Explicit injection WINS over the plugin `cache` capability (the
    * host knows best; prevents double-wrapping when the workspace also
    * declares a cache plugin). Undefined → identical to before.
    */
   remoteCache?: RemoteCacheLayer
+  /**
+   * The largest artifact, decoded, the local cache saves, ingests or
+   * restores; default 2 GiB. A seam for a test, which cannot produce 2 GiB
+   * of output to prove the refusal: the CLI never sets it, and no config
+   * or variable reaches it.
+   */
+  artifactCeiling?: number
 }
 
 export interface RunSummary {
   ok: boolean
   outcomes: TaskOutcome[]
+  /** The persistent tasks `RunOptions.holdPersistent` handed back; absent when none is running. */
+  persistent?: HeldPersistent
+}
+
+export interface HeldPersistent {
+  /** The held tasks' ids, `pkg#task`. */
+  ids: readonly string[]
+  /** Send every held task `signal` (default SIGTERM), then SIGKILL after the kill grace; resolves once each is gone. */
+  stop(signal?: 'SIGINT' | 'SIGTERM'): Promise<void>
 }

package/src/orchestrator/persistent.ts

@@ -0,0 +1,99 @@
+// End-of-run disposition of persistent children (dev servers, watchers).
+// `executeTask` spawns them and returns at "ready" without awaiting exit;
+// ownership moves to the run's registry, and this is the one place that
+// decides which of them outlive the graph and how the rest go down.
+
+import type { TaskNode } from '../graph/index.js'
+import { killGraceMs } from '../util/index.js'
+import { holdGroups, killTree, untilGroupsGone } from '../exec/index.js'
+
+type Child = ReturnType<typeof Bun.spawn>
+
+/**
+ * Grace after SIGTERMing the dependency-only persistent tasks before
+ * force-killing any that trap or ignore it — so a wedged mock server can't
+ * hang a normal run at completion. Well-behaved servers exit far under
+ * this, so the happy path never waits it out.
+ */
+const PERSISTENT_SHUTDOWN_GRACE_MS = 2000
+
+export interface KeepAlive {
+  nodes: TaskNode[]
+  children: Child[]
+}
+
+/**
+ * A persistent task the user REQUESTED (or one surfaced for display) is the
+ * run's whole purpose — it is left running and blocked on at the very end,
+ * after the summary. Only in the real CLI foreground: a custom logger or
+ * `handleSignals: false` (watch mode, embedders) expects `run()` to return,
+ * not to sit on a server.
+ */
+export function selectKeepAlive(
+  registry: ReadonlyMap<string, Child>,
+  nodes: ReadonlyMap<string, TaskNode>,
+  foreground: boolean,
+): KeepAlive {
+  const out: KeepAlive = { nodes: [], children: [] }
+  if (!foreground) return out
+  for (const [id, child] of registry) {
+    const n = nodes.get(id)
+    if (n !== undefined && (n.requested || n.surfaced === true)) {
+      out.nodes.push(n)
+      out.children.push(child)
+    }
+  }
+  return out
+}
+
+/**
+ * SIGTERM every persistent child not kept alive and wait for them, bounded:
+ * SIGTERM gives well-behaved servers (vite, next, esbuild --watch) a moment
+ * to clean up; past the grace every group with a member left is SIGKILLed,
+ * so the run's normal completion cannot hang on one that traps the signal.
+ * The GROUP, not the leader: a `server & wait` shell dies on the SIGTERM
+ * while a server that ignores it lives on, holding the task's pipe, and
+ * vx printed its summary and never exited (nx#8286 reproduced on vx,
+ * 2026-09-24).
+ */
+export async function shutdownPersistent(
+  registry: ReadonlyMap<string, Child>,
+  keepAlive: readonly Child[],
+  graceMs: number = killGraceMs(PERSISTENT_SHUTDOWN_GRACE_MS),
+): Promise<CrashedPersistent[]> {
+  const kept = new Set(keepAlive)
+  const crashed: CrashedPersistent[] = []
+  for (const [id, child] of registry) {
+    if (kept.has(child) || !hasEnded(child) || child.exitCode === 0) continue
+    crashed.push({ id, code: child.exitCode ?? child.signalCode ?? 'unknown' })
+  }
+  const dying = [...registry.values()].filter((c) => !kept.has(c))
+  if (dying.length === 0) return crashed
+  const letGo = holdGroups(dying)
+  try {
+    for (const child of dying) killTree(child, 'SIGTERM')
+    for (const child of await untilGroupsGone(dying, graceMs)) killTree(child, 'SIGKILL')
+    await Promise.allSettled(dying.map((c) => c.exited))
+  } finally {
+    letGo()
+  }
+  return crashed
+}
+
+/**
+ * A dependency-only persistent task that ended on its own, not by the stop
+ * at the end of the graph, and not cleanly. Its dependants may still have
+ * passed, and until item 892 the run did too: the server was reported
+ * `success` and its crash was said nowhere, while a requested server's
+ * crash failed the run.
+ */
+export interface CrashedPersistent {
+  id: string
+  /** The exit code, or the signal that killed it. */
+  code: number | string
+}
+
+/** The child has exited or been killed; Bun sets these once it reaps it. */
+export function hasEnded(child: Child): boolean {
+  return child.exitCode !== null || child.signalCode !== null
+}

package/src/orchestrator/placement.ts

@@ -0,0 +1,212 @@
+// Where each task runs: the placement of a graph over the resolved
+// executors, and the plan-mode view of it. Split from run.ts on 2026-09-10
+// (pure motion). A task is pinned to this machine when it is persistent,
+// depends on a persistent task, is sandboxed, or says `exec.remote: false`;
+// everything else asks the executors in declaration order, and the local
+// floor takes what nothing claimed.
+
+import { machineParallelism } from '../util/index.js'
+import { selectExecutor, type TaskExecutor } from '../exec/index.js'
+import { isGroupTask, type TaskNode } from '../graph/index.js'
+import { resolveDownloadModes } from './download-policy.js'
+import type { Logger } from './logger.js'
+import { resolveExecutors } from './plugin-host.js'
+import type { prepareRun } from './prepare.js'
+
+/**
+ * A task is pinned to this machine when it is persistent, transitively
+ * depends on a persistent task (a worker cannot reach a port on the
+ * submitter), declares `exec.sandbox` (the sandbox is this machine's
+ * machinery — a worker has none of it, and a boundary "verified" where it
+ * is not enforced passes vacuously), or declares `exec.remote: false`. A
+ * dependant of a pinned task is pinned with it.
+ */
+export function pinnedLocalSet(nodes: Map<string, TaskNode>): Set<string> {
+  const pinned = new Set<string>()
+  for (const node of nodes.values()) {
+    if (
+      node.config.exec?.persistent !== undefined ||
+      node.config.exec?.sandbox !== undefined ||
+      node.config.exec?.remote === false
+    ) {
+      pinned.add(node.id)
+    }
+  }
+  if (pinned.size === 0) return pinned
+  // Pinning flows up the dependant edges from what pins itself, one walk on
+  // an explicit stack: a chain is as deep as the graph, and a recursion
+  // per edge threw `RangeError` on the 50,000 the builder takes (item 737).
+  const dependants = new Map<string, string[]>()
+  for (const node of nodes.values()) {
+    for (const dep of node.deps) {
+      const list = dependants.get(dep)
+      if (list === undefined) dependants.set(dep, [node.id])
+      else list.push(node.id)
+    }
+  }
+  const stack = [...pinned]
+  while (stack.length > 0) {
+    for (const up of dependants.get(stack.pop()!) ?? []) {
+      if (pinned.has(up)) continue
+      pinned.add(up)
+      stack.push(up)
+    }
+  }
+  return pinned
+}
+
+export interface Placements {
+  executors: Map<string, TaskExecutor>
+  /**
+   * `exec.remote: 'only'` tasks that no REMOTE executor took — a NO-OP on
+   * this machine: never executed, declared outputs never cleaned or
+   * restored. The task exists to produce a remote input tree; without a
+   * remote pool, dependents use the machine's ambient state exactly as they
+   * did before the field existed.
+   */
+  remoteOnlyNoop: Set<string>
+  /** `'only'` tasks a remote executor DID take — executed remotely, outputs stay remote. */
+  remoteOnly: Set<string>
+}
+
+/**
+ * The tasks that write IN PLACE — everything not placed on a remote
+ * executor. `resolveDownloadModes` reads it as "the outputs are already
+ * here", so a task wrongly left out is one vx thinks it must fetch from a
+ * CAS that never held it.
+ *
+ * `remote` is a THREE-state field: `true` (remote), `false` (an explicit
+ * local plugin) and undefined — which is core's own local floor, since
+ * `localExecutor()` declares no `remote` at all. So the question is "not
+ * remote", never "declared local"; asking the latter drops every task on
+ * the floor executor, i.e. every task in a workspace with no executor
+ * plugin. It lives here, shared, because `run()` and `--dry` each built
+ * this set inline from the same predicate and a second copy is how the
+ * plan and the run would disagree about what comes home (441, 442, 445).
+ */
+export function locallyPlaced(placements: Placements, ids: Iterable<string>): Set<string> {
+  return new Set([...ids].filter((id) => placements.executors.get(id)?.remote !== true))
+}
+
+export function placeTasks(
+  nodes: Map<string, TaskNode>,
+  executors: readonly TaskExecutor[],
+): Placements {
+  const pinned = pinnedLocalSet(nodes)
+  const placements: Placements = {
+    executors: new Map(),
+    remoteOnlyNoop: new Set(),
+    remoteOnly: new Set(),
+  }
+  for (const node of nodes.values()) {
+    if (isGroupTask(node) || node.config.exec?.persistent !== undefined) continue
+    const executor = selectExecutor(executors, {
+      taskId: node.id,
+      projectName: node.projectName,
+      projectDir: node.projectDir,
+      command: node.config.exec!.command,
+      pinnedLocal: pinned.has(node.id),
+      cacheable: node.config.cache !== undefined,
+    })
+    placements.executors.set(node.id, executor)
+    if (node.config.exec?.remote === 'only') {
+      // A pinned 'only' task (it transitively depends on a persistent one)
+      // lands here too: pinning wins, so it noops rather than shipping.
+      if (executor.remote === true) placements.remoteOnly.add(node.id)
+      else placements.remoteOnlyNoop.add(node.id)
+    }
+  }
+  return placements
+}
+
+/**
+ * `executorOf` for `planRun`, or nothing. Declining plugins, a single
+ * executor, or a resolution error all yield nothing: `--dry` is an
+ * inspection command and must not fail over a label. The error is still
+ * said, on the status line, in the plugin's name: the run this plan
+ * previews would refuse on it, and a plan that hid that would read as
+ * "everything lands locally".
+ */
+export async function planExecutorOf(
+  prepared: Awaited<ReturnType<typeof prepareRun>>,
+  log: Logger,
+  policy: 'all' | 'toplevel' | 'none',
+): Promise<{
+  executorOf?: (id: string) => string | undefined
+  downloadOf?: (id: string) => 'eager' | 'deferred' | 'never' | undefined
+  downloadDowngrades?: ReadonlyArray<{ taskId: string; reason: string }>
+}> {
+  let executors: readonly TaskExecutor[]
+  try {
+    executors = await resolveExecutors(prepared.plugins, {
+      workspaceRoot: prepared.workspaceRoot,
+      cacheDir: prepared.cacheDir,
+      warn: (m: string) => log.status(m),
+      concurrency: machineParallelism(),
+    })
+  } catch (err) {
+    log.status(
+      `[vx] placement not shown — ${err instanceof Error ? err.message : String(err)} (the run would refuse on it)`,
+    )
+    return {}
+  }
+  const placements = placeTasks(prepared.nodes, executors)
+  // Download modes need placement regardless of how many executors there
+  // are (a single REMOTE one still defers); executor LABELS only earn their
+  // column when there is a choice to report.
+  const download =
+    policy === 'all'
+      ? undefined
+      : resolveDownloadModes({
+          nodes: prepared.nodes,
+          policy,
+          localPlaced: locallyPlaced(placements, prepared.nodes.keys()),
+          remoteOnly: placements.remoteOnly,
+        })
+  return {
+    ...(executors.length < 2
+      ? {}
+      : {
+          executorOf: (id: string) =>
+            placements.remoteOnlyNoop.has(id) ? 'noop' : placements.executors.get(id)?.name,
+        }),
+    ...(download === undefined
+      ? {}
+      : {
+          downloadOf: (id: string) => download.modeOf.get(id),
+          downloadDowngrades: [...download.downgrades].map(([taskId, reason]) => ({
+            taskId,
+            reason,
+          })),
+        }),
+  }
+}
+
+/**
+ * Stands in for the executor of a task that was never placed — a group task
+ * (runs nothing) or a persistent one (`executePersistentTask` owns it and
+ * never reads this field). It THROWS rather than silently picking some
+ * executor from the list, so a refactor that routes such a task through the
+ * exec path fails loudly instead of shipping a localhost server to a worker.
+ */
+export const UNPLACED_EXECUTOR: TaskExecutor = {
+  name: 'unplaced',
+  execute: (req) => {
+    throw new Error(`internal error: ${req.taskId} reached an executor without being placed`)
+  },
+}
+
+export function hasPooledExecutor(executors: readonly TaskExecutor[]): boolean {
+  return executors.some((e) => e.capacity !== undefined)
+}
+
+export function poolOfPlacement(
+  placements: Placements,
+): (id: string) => { name: string; capacity: number } | undefined {
+  return (id) => {
+    const executor = placements.executors.get(id)
+    return executor?.capacity === undefined
+      ? undefined
+      : { name: executor.name, capacity: executor.capacity }
+  }
+}

package/src/orchestrator/plan.ts

@@ -16,6 +16,7 @@ import { FULL_CACHE_POLICY } from '../cache/index.js'
 import { isGroupTask, runGraph, type TaskNode, type TaskOutcome } from '../graph/index.js'
 import type { HistoryProvider } from './history.js'
 import { computeGroupHash, computeTaskHash } from './task-hash.js'
+import { keyUpstream } from './upstream.js'
 
 export type CacheStatus =
   | 'hit-local' // entry exists in local cache
@@ -124,11 +125,20 @@ export async function plan(args: PlanArgs): Promise<RunPlan> {
   const outcomes = await runGraph({
     nodes: args.nodes,
     concurrency: 1,
-    execute: async (node, upstream) => {
+    execute: async (node, live) => {
+      const upstream = keyUpstream(node, live)
       if (isGroupTask(node)) {
         cacheStatusById.set(node.id, 'group')
         return planOutcome(node, computeGroupHash(upstream))
       }
+      // A persistent task has no key on the live path (its outcome carries
+      // none, `stable-keys.ts`), so a dependant folds nothing for it. The
+      // plan keyed it and folded that, and `--dry` then called a dependant
+      // a miss under a key the run never looks up (item 766).
+      if (node.config.exec?.persistent !== undefined) {
+        cacheStatusById.set(node.id, 'no-cache')
+        return planOutcome(node, undefined)
+      }
 
       const hash = await computeTaskHash({
         node,
@@ -142,17 +152,20 @@ export async function plan(args: PlanArgs): Promise<RunPlan> {
         ...(args.hashCache !== undefined ? { hashCache: args.hashCache } : {}),
       })
 
-      // Prediction keys off READS: a no-read policy (--no-cache /
-      // --force / --cache=local:,remote:) would re-execute every task,
-      // so we predict misses for it. The probe below routes through the
-      // policy-aware cache layer, which itself respects local/remote
-      // read gating.
+      // Prediction keys off READS: a no-read policy re-executes every task,
+      // so nothing is probed. Whether that run is a miss or no cache at all
+      // is the WRITE axes': `--force` and `--cache=local:w` save what they
+      // run, and the plan called them `no-cache`, the label a task with no
+      // `cache` block gets (item 991). The probe below routes through the
+      // policy-aware cache layer, which itself respects read gating.
       const policy = args.cachePolicy ?? FULL_CACHE_POLICY
-      const cacheEnabled =
-        node.config.cache !== undefined && (policy.localRead || policy.remoteRead)
+      const reads = policy.localRead || policy.remoteRead
+      const writes = policy.localWrite || policy.remoteWrite
       let status: CacheStatus
-      if (!cacheEnabled) {
+      if (node.config.cache === undefined || (!reads && !writes)) {
         status = 'no-cache'
+      } else if (!reads) {
+        status = 'miss'
       } else {
         const where = await args.cache.has(hash)
         status = where === null ? 'miss' : where === 'remote' ? 'hit-remote' : 'hit-local'
@@ -261,12 +274,12 @@ function predictPlan(tasks: PlannedTask[]): PlanPrediction {
   return { wallMs, workMs, unknownCount }
 }
 
-function planOutcome(node: TaskNode, hash: string): TaskOutcome {
+function planOutcome(node: TaskNode, hash: string | undefined): TaskOutcome {
   return {
     node,
     status: 'success',
     exitCode: 0,
     durationMs: 0,
-    hash,
+    ...(hash !== undefined ? { hash } : {}),
   }
 }