@vzn/vx 0.0.16 → 0.0.21

package/src/orchestrator/admission.ts

@@ -0,0 +1,137 @@
+// What stands between the scheduler's `execute` callback and executeTask:
+// two rules that decide whether a ready task runs NOW and whether its
+// result may be SAVED. Neither changes what the task is.
+//
+// - In-flight dedup: an embedder running concurrent delegated runs in one
+//   process (a daemon built on the façade; core ships none) supplies an
+//   `inflight` registry; a task whose key
+//   a sibling is already computing waits for it and cache-hits on what it
+//   saved. A stateless `vx run` passes none and takes the untouched path.
+// - Continue-taint: under `continueMode: 'always'` a task runs although an
+//   upstream failed. Its key is the healthy one (pure-input hashing) but
+//   its bytes are not, so the taint is tracked here and propagates through
+//   every success built on it — or a grand-dependent would cache the same
+//   partial tree one hop later. Only that mode ever executes a task behind
+//   a failure; the other modes skip it, so a default run carries no check.
+//
+// Split from `run.ts` on 2026-09-10 (pure motion).
+
+import type { CachePolicy } from '../cache/index.js'
+import { isGroupTask, type TaskNode, type TaskOutcome } from '../graph/index.js'
+import { executeTask, type ExecuteArgs } from './execute-task.js'
+import type { ShortCircuit } from './local-shortcircuit.js'
+import { computeTaskHash, type ComputeHashArgs } from './task-hash.js'
+
+/**
+ * Records which tasks ran behind a failure and answers, per task, whether
+ * this one does. Disabled (always `false`, nothing recorded) unless the
+ * run's `continueMode` is `'always'`.
+ */
+export function taintTracker(
+  enabled: boolean,
+): (node: TaskNode, upstream: TaskOutcome[]) => boolean {
+  if (!enabled) return () => false
+  const tainted = new Set<string>()
+  return (node, upstream) => {
+    const taint = upstream.some(
+      // A restore-tier task may run before its deps and see holes here;
+      // it never saves anyway (a hit restores), so a hole is not taint.
+      (u) =>
+        u !== undefined &&
+        (u.status === 'failed' ||
+          u.status === 'aborted' ||
+          u.status === 'skipped' ||
+          tainted.has(u.node.id)),
+    )
+    if (taint) tainted.add(node.id)
+    return taint
+  }
+}
+
+export interface AdmissionArgs {
+  /** The shared registry, or undefined for a stateless run (no dedup). */
+  inflight: Map<string, Promise<void>> | undefined
+  policy: CachePolicy
+  shortCircuit: ShortCircuit
+  /** Everything `computeTaskHash` needs besides the node and its upstream. */
+  hashArgs: Omit<ComputeHashArgs, 'node' | 'upstream' | 'nestedProjectDirs'> & {
+    nestedDirsByProject: ReadonlyMap<string, string[]>
+  }
+  /**
+   * The run's execute-args builder. `reuseProbe: false` drops the up-front
+   * probe: a task joining a sibling must let executeTask probe fresh,
+   * because the probe predates the sibling's save.
+   */
+  buildExecuteArgs: (node: TaskNode, upstream: TaskOutcome[], reuseProbe?: boolean) => ExecuteArgs
+}
+
+/** The scheduler's `execute` callback: dedup when it can help, else executeTask. */
+export function admitTasks(
+  args: AdmissionArgs,
+): (node: TaskNode, upstream: TaskOutcome[]) => Promise<TaskOutcome> {
+  const { inflight, policy, shortCircuit, hashArgs, buildExecuteArgs } = args
+  return async (node, upstream) => {
+    // Dedup only helps when the sibling will WRITE the artifact and this
+    // task can READ it back — i.e. both axes effectively on.
+    const canRead = policy.localRead || policy.remoteRead
+    const canWrite = policy.localWrite || policy.remoteWrite
+    const cacheable =
+      !isGroupTask(node) &&
+      node.config.exec?.persistent === undefined &&
+      node.config.cache !== undefined &&
+      canRead &&
+      canWrite
+    // A restore-tier task (confirmed local hit, may run before its deps)
+    // needs no dedup — it's a restore, not an executor, and its live
+    // `upstream` is incomplete, so the dedup hash recompute would be
+    // wrong. Route it straight to executeTask, which reuses the up-front
+    // probe.
+    const restorable = shortCircuit.restoreTier.has(node.id)
+    if (inflight === undefined || !cacheable || restorable) {
+      return executeTask(buildExecuteArgs(node, upstream))
+    }
+    const { nestedDirsByProject, ...rest } = hashArgs
+    const hash = await computeTaskHash({
+      ...rest,
+      node,
+      upstream,
+      nestedProjectDirs: nestedDirsByProject.get(node.projectName) ?? [],
+    })
+    const existing = inflight.get(hash)
+    if (existing !== undefined) {
+      // Join a sibling already computing this exact task: wait, then
+      // executeTask cache-hits on the artifact it just saved.
+      await existing.catch(() => {})
+      return executeTask(buildExecuteArgs(node, upstream, false))
+    }
+    // Become the executor: register a barrier siblings await. get→set has
+    // no await between, so registration is atomic — at most one executor
+    // per hash. Released on every exit (success / failure / throw).
+    let release!: () => void
+    inflight.set(
+      hash,
+      new Promise<void>((resolve) => {
+        release = resolve
+      }),
+    )
+    const execArgs = buildExecuteArgs(node, upstream)
+    try {
+      // `return await`, deliberately: the finally must run after the task
+      // settles, not when its promise is handed back.
+      return await executeTask(execArgs)
+    } finally {
+      // The barrier lifts when the ENTRY is there, not when the task is:
+      // the save runs off the slot (save-lane.ts), and a sibling released
+      // before it landed would probe a miss and run the task again. The
+      // executor's own return is not held — only the joiners wait.
+      const landed = execArgs.deferredSaves?.get(node.id)
+      const lift = (): void => {
+        execArgs.deferredSaves?.delete(node.id)
+        inflight.delete(hash)
+        release()
+      }
+      if (landed !== undefined) void landed.then(lift, lift)
+      else lift()
+    }
+  }
+}

package/src/orchestrator/deferred-outputs.ts

@@ -0,0 +1,147 @@
+// The run-scoped registry of tasks whose outputs were left in the remote
+// store (`--download=none`), and the lazy materialisation that fetches
+// them when a locally-placed task turns out to need them.
+//
+// See docs/design/download-policy-cas-cache-2026-08.md §5. The two
+// invariants that keep this from becoming a second cache:
+//
+//   - A deferred task writes NOTHING locally — no artifact, no rows. The
+//     remote store (the executor's own record) is the entry.
+//   - Materialisation CONVERGES: after the bytes land, core runs the
+//     ordinary `cache.save`, so the machine is indistinguishable from a
+//     `--download=all` run and every later reader (get, prune, verify,
+//     metrics) sees one entry shape, not two.
+
+import {
+  cleanOutputs,
+  resolveOutputs,
+  type CacheLayer,
+  type GitFilesCache,
+} from '../cache/index.js'
+import type { TaskNode } from '../graph/index.js'
+
+export interface DeferredEntry {
+  /** Fetches this task's outputs onto disk. At-most-once by construction. */
+  materialize: () => Promise<void>
+  hash: string
+  entry: { taskId: string; command: string; durationMs: number; stdout: string }
+}
+
+export interface DeferredOutputsArgs {
+  nodes: Map<string, TaskNode>
+  cache: CacheLayer
+  workspaceRoot: string
+  nestedDirsByProject: Map<string, string[]>
+  gitFilesCache?: GitFilesCache
+  /** False when the run's policy writes no local entries. */
+  localWrite: boolean
+}
+
+export class DeferredOutputs {
+  private readonly entries = new Map<string, DeferredEntry>()
+  private readonly inflight = new Map<string, Promise<void>>()
+
+  constructor(private readonly args: DeferredOutputsArgs) {}
+
+  register(taskId: string, entry: DeferredEntry): void {
+    this.entries.set(taskId, entry)
+  }
+
+  /**
+   * Task ids whose outputs are still remote — the summary's list. An entry
+   * is removed only when materialisation SUCCEEDS, so this covers both
+   * "nothing needed them" and "fetching them failed"; the second is
+   * precisely when a user needs to be told the tree is not current, and an
+   * `inflight` filter here used to hide it.
+   */
+  pending(): string[] {
+    return [...this.entries.keys()].sort()
+  }
+
+  get size(): number {
+    return this.entries.size
+  }
+
+  /**
+   * Materialise every deferred producer in `node`'s TRANSITIVE dependency
+   * closure. Which upstream bytes a command reads is unknowable — that is
+   * what `dependsOn` declares — so the whole closure is taken. Each
+   * producer materialises at most once per run (memoised on the promise,
+   * so two concurrent consumers share one fetch), and they run
+   * concurrently.
+   *
+   * A failure is the CONSUMER's failure: executing against a
+   * half-materialised tree is the stale-input class with extra steps, so
+   * this throws and the caller fails the task naming the producer.
+   */
+  async materializeFor(node: TaskNode): Promise<void> {
+    const needed: string[] = []
+    const seen = new Set<string>()
+    const walk = (id: string): void => {
+      for (const dep of this.args.nodes.get(id)?.deps ?? []) {
+        if (seen.has(dep)) continue
+        seen.add(dep)
+        if (this.entries.has(dep)) needed.push(dep)
+        walk(dep)
+      }
+    }
+    walk(node.id)
+    if (needed.length === 0) return
+    await Promise.all(needed.map((id) => this.materializeOne(id)))
+  }
+
+  private materializeOne(taskId: string): Promise<void> {
+    const existing = this.inflight.get(taskId)
+    if (existing !== undefined) return existing
+    const started = this.run(taskId)
+    this.inflight.set(taskId, started)
+    return started
+  }
+
+  private async run(taskId: string): Promise<void> {
+    const entry = this.entries.get(taskId)!
+    const producer = this.args.nodes.get(taskId)!
+    // Project outputs only: a task declaring `outputs.workspaceFiles` is
+    // never deferred (`deferralEligibility` forces it eager), so nothing
+    // workspace-anchored can reach this path.
+    const outputs = producer.config.cache?.outputs.files ?? []
+    const nestedProjectDirs = this.args.nestedDirsByProject.get(producer.projectName) ?? []
+    const cleanArgs = { projectDir: producer.projectDir, outputs, nestedProjectDirs }
+
+    // Mirror `restoreHit`'s sequencing so the two paths cannot drift: wipe
+    // the declared outputs, write, then tell the git snapshot what moved.
+    if (outputs.length > 0) {
+      const cleaned = await cleanOutputs(cleanArgs)
+      this.args.gitFilesCache?.markOutputsChanged(producer.projectDir, cleaned)
+    }
+    try {
+      await entry.materialize()
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err)
+      throw new Error(
+        `could not fetch deferred outputs of ${taskId}: ${msg} — re-run it with --force, or use --download=all`,
+      )
+    }
+    const outputFiles = await resolveOutputs({
+      projectDir: producer.projectDir,
+      outputs,
+      nestedProjectDirs,
+    })
+    this.args.gitFilesCache?.markOutputsChanged(
+      producer.projectDir,
+      outputFiles.map((f) => f.slice(producer.projectDir.length + 1)),
+    )
+    // Convergence: an ordinary entry, so the next run is a plain local hit
+    // and nothing downstream learns a second shape. Policy still governs —
+    // a run that writes no local entries writes none here either.
+    if (this.args.localWrite) {
+      await this.args.cache.save({
+        hash: entry.hash,
+        projectDir: producer.projectDir,
+        outputFiles,
+        entry: entry.entry,
+      })
+    }
+    this.entries.delete(taskId)
+  }
+}

package/src/orchestrator/doctor.ts

@@ -0,0 +1,255 @@
+// The workspace doctor's facts — what `vx info` prints and `vx mcp`'s
+// `getWorkspaceInfo` returns: versions, the git status cache, the
+// workspace's shape, the plugins and the seams each fills, what a run
+// will use (workers, memory) and where those come from, the cache's
+// versions and state, the flaky tasks, the lock. One collector, so the
+// verb and the agent tool cannot disagree; the CLI renders, this gathers.
+
+import os from 'node:os'
+import path from 'node:path'
+import { Cache, CACHE_VERSION, noteSchemaReset, SCHEMA_VERSION } from '../cache/index.js'
+import { PLUGIN_HOOKS } from '../config.js'
+import {
+  cgroupCpuQuota,
+  cgroupMemoryLimitBytes,
+  machineMemoryBytes,
+  machineParallelism,
+} from '../util/index.js'
+import { VERSION } from '../version.js'
+import {
+  buildPackageGraph,
+  computeWorkspaceFingerprint,
+  findWorkspaceRoot,
+  listProjects,
+  loadProjectConfig,
+  loadWorkspace,
+  lockfilePath,
+  type ProjectMeta,
+  resolveCacheDir,
+} from '../workspace/index.js'
+import { flakyTasks, type FlakyTask } from './failure-mode.js'
+import type { VxPlugin } from './plugin.js'
+import { loadProjects, loadWorkspacePlugins } from './projects.js'
+
+const warnToStderr = (message: string): void => {
+  process.stderr.write(`${message}\n`)
+}
+
+/** The doctor's facts, typed: what `--format json` prints and the pretty rows render. */
+export interface InfoFacts {
+  vx: string
+  bun: string
+  git: string | null
+  /** null when git could not answer. */
+  gitStatusCache: { fsmonitor: boolean; untrackedCache: boolean } | null
+  workspaceRoot: string
+  projects: number
+  tasks: number
+  plugins: Array<{ name: string; seams: string[] }>
+  /**
+   * The worker count a run defaults to and where it comes from: the
+   * workspace's `concurrency`, else the cores this process may use — the
+   * CPU count capped by a cgroup quota (`cpuQuota`, in cores, null when
+   * none binds).
+   */
+  workers: {
+    count: number
+    source: 'workspace' | 'cgroup' | 'cores'
+    cores: number
+    cpuQuota: number | null
+  }
+  /**
+   * What a memory-packing policy budgets: the machine's total capped by
+   * the cgroup limit (`cgroupLimitBytes`, null when none binds).
+   */
+  memory: { usableBytes: number; totalBytes: number; cgroupLimitBytes: number | null }
+  cacheDir: string
+  cacheVersion: string
+  schemaVersion: string
+  cacheEntries: number
+  cacheBytes: number
+  orphans: { artifacts: number; bytes: number }
+  runs24h: number
+  hits24h: number
+  /** Tasks the retained history shows both passing and failing on unchanged inputs. */
+  flakyTasks: FlakyTask[]
+  lockfile: boolean
+}
+
+export interface CollectInfoOptions {
+  /** `--cache-dir`: report on the cache a run with the same flag uses. */
+  readonly cacheDir?: string
+  /** Where a warning (a schema reset, a plugin's note) goes. Default: stderr. */
+  readonly warn?: (message: string) => void
+}
+
+export async function collectInfo(cwd: string, opts: CollectInfoOptions = {}): Promise<InfoFacts> {
+  const warn = opts.warn ?? warnToStderr
+  const root = await findWorkspaceRoot(cwd)
+  const metas = await listProjects(await loadWorkspace(root))
+  const { workspaceConfig, plugins } = await loadWorkspacePlugins(root, warn)
+  const cacheDir =
+    opts.cacheDir === undefined
+      ? resolveCacheDir(root, workspaceConfig)
+      : path.resolve(cwd, opts.cacheDir)
+  const cache = new Cache(cacheDir)
+  noteSchemaReset(cache, warn)
+  let stats
+  let orphans
+  let flaky: FlakyTask[]
+  let taskCount = 0
+  try {
+    stats = cache.stats()
+    orphans = await cache.orphanStats()
+    flaky = flakyTasks(cache.dbHandle())
+    // The run path's load — a plugin's `project` stage counts — so the
+    // doctor's task count is the number a run would see. A broken config
+    // must not take the doctor down with it: the count then falls back to
+    // the configs that do load, one by one, the broken ones as zero.
+    try {
+      const loaded = await loadProjects({
+        workspaceRoot: root,
+        cacheDir,
+        plugins,
+        projectMetas: metas,
+        packageGraph: buildPackageGraph([...metas]),
+        seeds: 'all',
+        closure: false,
+        lock: null,
+        evalCache: { store: cache, workspaceFingerprint: await computeWorkspaceFingerprint(root) },
+        warn,
+      })
+      for (const p of loaded.projects.values()) {
+        taskCount += Object.keys(p.config.tasks ?? {}).length
+      }
+    } catch {
+      taskCount = await countLoadableTasks(metas)
+    }
+  } finally {
+    cache.close()
+  }
+
+  const lockPresent = await Bun.file(lockfilePath(root)).exists()
+  return {
+    vx: VERSION,
+    bun: Bun.version,
+    git: gitVersion(),
+    // The one `git status` walk per run is the warm path's critical path on
+    // a large tree (~55 ms at 1000 projects, measured 2026-09-02). git's
+    // own caches make it near-free after the first run, and they are OFF by
+    // default — say so, since nothing else in a run would.
+    gitStatusCache: gitStatusCache(root),
+    workspaceRoot: root,
+    projects: metas.length,
+    tasks: taskCount,
+    // Which plugins loaded and which seams each fills, in pipeline order —
+    // the answer to "why did this task run there / cache there / not at
+    // all" before reading any config. A declined seam still costs nothing;
+    // this names the declarations, not what a run consulted.
+    plugins: plugins.map((p) => ({ name: p.name, seams: filledSeams(p) })),
+    workers: workersFact(workspaceConfig?.concurrency),
+    memory: memoryFact(),
+    cacheDir,
+    // The two versions a bug report needs and the reset notice names: the
+    // key prefix (a bump orphans every entry) and the index schema (a
+    // mismatch drops every table).
+    cacheVersion: CACHE_VERSION,
+    schemaVersion: SCHEMA_VERSION,
+    cacheEntries: stats.entryCount,
+    cacheBytes: stats.totalBytes,
+    // The index is authoritative, so a row-less artifact is bytes nothing
+    // will ever hit — and only `vx cache prune` reclaims them (after an
+    // upgrade's schema reset, most often).
+    orphans: { artifacts: orphans.orphans, bytes: orphans.orphanBytes },
+    runs24h: stats.runCountLast24h,
+    hits24h: stats.hitCountLast24h,
+    // Same inputs, both outcomes — the history's definition of flaky, over
+    // the 30 days it keeps. A run names its own findings in its footer;
+    // this is the workspace's standing list.
+    flakyTasks: flaky,
+    lockfile: lockPresent,
+  }
+}
+
+/** The seams a plugin can fill, in pipeline order: the one hook list, less the lifecycle end. */
+const SEAMS = PLUGIN_HOOKS.filter((h) => h !== 'teardown')
+
+function filledSeams(p: VxPlugin): string[] {
+  return SEAMS.filter((s) => p[s as keyof VxPlugin] !== undefined)
+}
+
+/**
+ * Inside a container the CPU count and `os.totalmem()` are the HOST's; the
+ * cgroup is what the kernel enforces, and a run that used the host's
+ * numbers would be the OOM killer's. The doctor says which one a run
+ * reads, because nothing else would.
+ */
+function workersFact(workspaceConcurrency: number | undefined): InfoFacts['workers'] {
+  const cores = Math.max(1, navigator.hardwareConcurrency)
+  const quota = process.platform === 'linux' ? (cgroupCpuQuota() ?? null) : null
+  if (workspaceConcurrency !== undefined) {
+    return { count: workspaceConcurrency, source: 'workspace', cores, cpuQuota: quota }
+  }
+  const machine = machineParallelism()
+  return { count: machine, source: machine < cores ? 'cgroup' : 'cores', cores, cpuQuota: quota }
+}
+
+function memoryFact(): InfoFacts['memory'] {
+  const totalBytes = os.totalmem()
+  const limit = process.platform === 'linux' ? (cgroupMemoryLimitBytes() ?? null) : null
+  return { usableBytes: machineMemoryBytes(), totalBytes, cgroupLimitBytes: limit }
+}
+
+/**
+ * Whether git's fsmonitor / untracked cache are on. Reported as a fact,
+ * not a remedy: interleaved A/B at 1000 projects measured neither moving
+ * the warm run (STATUS, waves 5 and the 2026-09-03 refutations) — the
+ * status walk's cost is git's own, and vx already overlaps it.
+ */
+function gitStatusCache(root: string): InfoFacts['gitStatusCache'] {
+  try {
+    const p = Bun.spawnSync({
+      cmd: ['git', 'config', '--get-regexp', '^core\\.(fsmonitor|untrackedcache)$'],
+      cwd: root,
+      stdout: 'pipe',
+      stderr: 'pipe',
+    })
+    // exit 1 is git's "no key matched": both off, still an answer.
+    if (p.exitCode !== 0 && p.exitCode !== 1) return null
+    const out = p.exitCode === 0 ? new TextDecoder().decode(p.stdout) : ''
+    const on = (key: string): boolean =>
+      new RegExp(`^core\\.${key} (true|1|yes|on)$`, 'im').test(out)
+    return { fsmonitor: on('fsmonitor'), untrackedCache: on('untrackedcache') }
+  } catch {
+    return null
+  }
+}
+
+function gitVersion(): string | null {
+  try {
+    const p = Bun.spawnSync({ cmd: ['git', '--version'], stdout: 'pipe', stderr: 'pipe' })
+    if (p.exitCode !== 0) return null
+    return new TextDecoder()
+      .decode(p.stdout)
+      .trim()
+      .replace(/^git version /, '')
+  } catch {
+    return null
+  }
+}
+
+async function countLoadableTasks(metas: readonly ProjectMeta[]): Promise<number> {
+  let count = 0
+  await Promise.all(
+    metas.map(async (meta) => {
+      if (meta.configPath === null) return
+      try {
+        const config = await loadProjectConfig(meta.configPath)
+        count += Object.keys(config.tasks ?? {}).length
+      } catch {
+        // counted as zero
+      }
+    }),
+  )
+  return count
+}