@vzn/vx 0.0.18 → 0.0.22

package/src/orchestrator/projects.ts

@@ -0,0 +1,270 @@
+// The project-config load every consumer of "what tasks exist" shares:
+// `prepareRun` (a run, a plan) and `vx show` (introspection). One code
+// path, so what `show` prints is what a run would see — including the
+// plugin `project` stage, which can give tasks to a package that never
+// wrote a `vx.config.ts`.
+
+import type { ProjectConfig } from '../config.js'
+import {
+  frozenProjectConfig,
+  loadProjectConfigs,
+  validateProjectConfig,
+  type LoadProjectConfigOptions,
+  type LoadReads,
+  type Lockfile,
+  type PackageGraph,
+  type ProjectEntry,
+  type ProjectMeta,
+} from '../workspace/index.js'
+import type { WorkspaceConfig } from '../config.js'
+import { Cache, noteSchemaReset } from '../cache/index.js'
+import {
+  buildPackageGraph,
+  computeWorkspaceFingerprint,
+  listProjects,
+  loadWorkspace,
+  loadWorkspaceConfig,
+  resolveCacheDir,
+} from '../workspace/index.js'
+import { parseDependencySpec } from '../graph/index.js'
+import { applyConfigHooks, applyProjectHooks, hasHook } from './plugin-host.js'
+import type { VxPlugin } from './plugin.js'
+
+/**
+ * The workspace config with the plugin `config` stage applied — the first
+ * stage of the pipeline, run before anything is derived from the config.
+ * The plugin list itself is already fixed by then.
+ */
+export async function loadWorkspacePlugins(
+  workspaceRoot: string,
+  warn: (message: string) => void,
+): Promise<{ workspaceConfig: WorkspaceConfig | null; plugins: readonly VxPlugin[] }> {
+  const workspaceConfig = await loadWorkspaceConfig(workspaceRoot)
+  const plugins = (workspaceConfig?.plugins ?? []) as readonly VxPlugin[]
+  if (workspaceConfig !== null && hasHook(plugins, 'config')) {
+    await applyConfigHooks(plugins, workspaceConfig, { workspaceRoot, warn })
+  }
+  return { workspaceConfig, plugins }
+}
+
+export interface LoadProjectsArgs {
+  workspaceRoot: string
+  cacheDir: string
+  plugins: readonly VxPlugin[]
+  projectMetas: readonly ProjectMeta[]
+  packageGraph: PackageGraph
+  /**
+   * Projects whose configs must load — `'all'`, or names (unknown and
+   * config-less ones are ignored). With `closure`, each seed's transitive
+   * package dependencies load too, which bounds `^task` frontier
+   * expansion; a `pkg#task` dependsOn entry pulls its project in either
+   * way, since the package graph cannot see the cross form.
+   */
+  seeds: 'all' | Iterable<string>
+  closure: boolean
+  /** Read configs from the lock instead of evaluating them (`--frozen`). */
+  lock: Lockfile | null
+  evalCache: LoadProjectConfigOptions['evalCache']
+  warn: (message: string) => void
+  /**
+   * Entries a load in this same process already produced with the same
+   * cache dir and lock (the CLI's selection pass, which stages every
+   * config to walk `pkg#task` edges). Seeding and scoping run exactly as
+   * without it; a project found here is taken as is instead of being
+   * evaluated and put through the `project` stage a second time — the
+   * stage's cost, and its warnings, once per run. Never carried across
+   * runs: a config can change between them.
+   */
+  staged?: ReadonlyMap<string, ProjectEntry>
+}
+
+export interface LoadedProjects {
+  projects: Map<string, ProjectEntry>
+  /**
+   * Every project that can carry tasks: it has a config file, or a plugin
+   * fills the `project` stage and may give it tasks. Boundary geometry
+   * fences all of them, loaded or not.
+   */
+  configured: readonly ProjectMeta[]
+}
+
+/**
+ * SCOPED config loading: configs are programs, and evaluating 1090 of
+ * them costs ~200 ms — the dominant fixed cost of small runs. Only the
+ * seeds, their closure and any project a `pkg#task` entry names can
+ * contribute graph nodes, so only those configs are evaluated. Side
+ * effect, deliberate and Turbo-like: a broken config in an unrelated
+ * package does not fail a scoped run — it surfaces when that package
+ * enters scope.
+ */
+export async function loadProjects(args: LoadProjectsArgs): Promise<LoadedProjects> {
+  const { plugins, packageGraph, lock, workspaceRoot } = args
+  // A package with no config file declares no tasks — unless a plugin
+  // fills the `project` stage, in which case it is a project the stage may
+  // give tasks to (the zero-migration shape: `turbo.json` or `package.json`
+  // scripts mapped onto packages that never wrote a `vx.config.ts`). It
+  // then loads as `{ tasks: {} }` — nothing to evaluate, nothing to freeze
+  // — and the stage runs on that like on any loaded config. With no
+  // `project` plugin a plain run never visits (or fences, or seeds) a
+  // config-less package.
+  const projectStage = hasHook(plugins, 'project')
+  const configured = args.projectMetas.filter(
+    (m) => projectStage || (typeof m.configPath === 'string' && m.configPath.length > 0),
+  )
+  const metaByName = new Map<string, ProjectMeta>(configured.map((m) => [m.name, m]))
+  const needed = new Set<string>()
+  const pending: ProjectMeta[] = []
+  // Counted, not read off `pending`: a round empties `pending`, and a guard
+  // on its length let every cross edge after the first round walk its
+  // whole closure again (1,500 walks at 300 projects, item 932).
+  let neededConfigured = 0
+  const consider = (name: string): void => {
+    if (needed.has(name)) return
+    needed.add(name)
+    const meta = metaByName.get(name)
+    if (meta) {
+      pending.push(meta)
+      neededConfigured++
+    }
+  }
+  const walked = new Set<string>()
+  const considerWithDeps = (name: string): void => {
+    consider(name)
+    // Every config-bearing project already considered: the closure can
+    // add nothing, and asking for it would build the package graph's
+    // transitive bitsets — a cost the unscoped run (every project a seed)
+    // otherwise never pays. A closure already walked adds nothing either.
+    if (neededConfigured === metaByName.size || walked.has(name)) return
+    walked.add(name)
+    for (const dep of packageGraph.transitiveDeps(name)) consider(dep)
+  }
+  const seeds = args.seeds === 'all' ? metaByName.keys() : [...args.seeds]
+  for (const seed of seeds) consider(seed)
+  if (args.closure && pending.length < metaByName.size) {
+    for (const seed of args.seeds === 'all' ? metaByName.keys() : seeds) considerWithDeps(seed)
+  }
+
+  // Load in rounds to a fixpoint. A `pkg#task` dependsOn entry names a
+  // project the PACKAGE graph cannot reach (the cross form ignores npm
+  // deps by design), so its config has to be pulled in — and that config
+  // may declare cross edges of its own. The common case (no cross deps)
+  // is a single round, identical to loading the closure in one batch.
+  const projects = new Map<string, ProjectEntry>()
+  while (pending.length > 0) {
+    const round = pending.splice(0, pending.length)
+    // Only a config FILE is evaluated (or read from the lock); a
+    // config-less project in the round starts from an empty task table,
+    // and a project the caller already staged is neither.
+    const withFile = round.filter(
+      (m): m is ProjectMeta & { configPath: string } =>
+        typeof m.configPath === 'string' && args.staged?.get(m.name) === undefined,
+    )
+    const loaded = lock
+      ? await Promise.all(withFile.map((m) => frozenProjectConfig(lock, m, workspaceRoot)))
+      : await loadProjectConfigs(
+          withFile.map((m) => m.configPath),
+          args.evalCache !== undefined ? { evalCache: args.evalCache } : {},
+        )
+    let next = 0
+    for (const meta of round) {
+      const pre = args.staged?.get(meta.name)
+      if (pre !== undefined) {
+        projects.set(meta.name, pre)
+        for (const name of crossDepProjects(pre.config)) {
+          if (args.closure) considerWithDeps(name)
+          else consider(name)
+        }
+        continue
+      }
+      const config: ProjectConfig =
+        meta.configPath === null ? { tasks: {} } : (loaded[next++] as ProjectConfig)
+      if (projectStage) {
+        // The `project` stage edits the validated object in place; core
+        // re-validates after EACH plugin so a plugin can only produce what
+        // the loader accepts from a user, and the refusal names both the
+        // config file and the plugin whose edit broke it.
+        const where = meta.configPath ?? `${meta.name} (no config file)`
+        await applyProjectHooks(
+          plugins,
+          config,
+          {
+            workspaceRoot,
+            cacheDir: args.cacheDir,
+            warn: args.warn,
+            name: meta.name,
+            dir: meta.dir,
+            packageJson: meta.packageJson as unknown as Readonly<Record<string, unknown>>,
+            projects: args.projectMetas,
+          },
+          (plugin) => validateProjectConfig(config, `${where} (after plugin '${plugin.name}')`),
+        )
+      }
+      projects.set(meta.name, { name: meta.name, dir: meta.dir, config })
+      for (const name of crossDepProjects(config)) {
+        if (args.closure) considerWithDeps(name)
+        else consider(name)
+      }
+    }
+  }
+  return { projects, configured }
+}
+
+/** Project names named by a `pkg#task` dependsOn entry anywhere in `config`. */
+function crossDepProjects(config: ProjectConfig): string[] {
+  const out: string[] = []
+  for (const task of Object.values(config.tasks ?? {})) {
+    for (const raw of task.dependsOn ?? []) {
+      // A malformed spec is the graph builder's error to report — it names
+      // the offending task. Here it just contributes no project.
+      if (!raw.includes('#')) continue
+      try {
+        const spec = parseDependencySpec(raw)
+        if (spec.kind === 'cross') out.push(spec.project)
+      } catch {
+        continue
+      }
+    }
+  }
+  return out
+}
+
+/**
+ * The run path's view of a workspace's projects for a reader — `vx show`,
+ * the MCP server, an embedder: discovery, the plugin `config` and
+ * `project` stages, and the local cache opened only to serve cached
+ * evaluations, so a pure config costs a stat, not an evaluation. `scope`
+ * is every project or a list of names; no closure, no lock (a reader
+ * reads live, as a default run does). Plugin warnings go to `warn`.
+ */
+export async function loadResolvedProjects(
+  workspaceRoot: string,
+  opts: { scope?: 'all' | readonly string[]; warn?: (message: string) => void } = {},
+): Promise<Map<string, ProjectEntry>> {
+  const warn = opts.warn ?? ((): void => {})
+  const reads: LoadReads = new Map()
+  const metas = await listProjects(await loadWorkspace(workspaceRoot, reads))
+  const { workspaceConfig, plugins } = await loadWorkspacePlugins(workspaceRoot, warn)
+  const cacheDir = resolveCacheDir(workspaceRoot, workspaceConfig)
+  const cache = new Cache(cacheDir, { read: true, write: true }, workspaceRoot)
+  noteSchemaReset(cache, warn)
+  try {
+    const loaded = await loadProjects({
+      workspaceRoot,
+      cacheDir,
+      plugins,
+      projectMetas: metas,
+      packageGraph: buildPackageGraph([...metas]),
+      seeds: opts.scope ?? 'all',
+      closure: false,
+      lock: null,
+      evalCache: {
+        store: cache,
+        workspaceFingerprint: await computeWorkspaceFingerprint(workspaceRoot, reads),
+      },
+      warn,
+    })
+    return loaded.projects
+  } finally {
+    cache.close()
+  }
+}

package/src/orchestrator/remote-prefetch.ts

@@ -70,7 +70,7 @@ async function runPrefetch(args: PrefetchArgs): Promise<void> {
   // tells us which of the N stable hashes exist remotely, so we GET only the
   // hits and pre-mark the misses — their lazy `get` then short-circuits with
   // no network. This collapses N probe waves into 1 and skips every GET that
-  // would 404. When the remote can't batch (`null` — an older serve, reads
+  // would 404. When the remote can't batch (`null` — an older remote, reads
   // disabled, or a layer that doesn't implement the hook at all), fall back to
   // prefetching every stable key, exactly as before.
   let toPrefetch = stableKeys
@@ -78,6 +78,11 @@ async function runPrefetch(args: PrefetchArgs): Promise<void> {
   const present = (await args.cache.remoteHasMany?.(uniqueHashes)) ?? null
   if (present !== null) {
     args.cache.markRemoteAbsent?.(uniqueHashes.filter((h) => !present.has(h)))
+    // The pre-mark above already answers a pull for an absent hash from
+    // the in-flight map, so on the wire this filter is the pump's
+    // economy: deleting it alone GETs nothing extra, deleting both GETs
+    // every absent key. What it decides is which hashes the pump asks
+    // the layer for, and that is what the row pins (item 643, 645).
     toPrefetch = stableKeys.filter((k) => present.has(k.hash))
     if (toPrefetch.length === 0) return
   }
@@ -92,9 +97,24 @@ async function runPrefetch(args: PrefetchArgs): Promise<void> {
     while (next < toPrefetch.length) {
       const { hash, node } = toPrefetch[next++]!
       await args.cache
-        .prefetch(hash, { taskId: node.id, command: node.config.exec?.command ?? '' })
+        .prefetch(hash, getContext(node, node.config.exec?.command ?? ''))
         .catch(() => false)
     }
   }
   await Promise.all(Array.from({ length: workers }, () => pump()))
 }
+
+/**
+ * What a cache lookup is told about the task: its id and command for the
+ * row a remote hit inserts, and its declared outputs, the only names a
+ * remote artifact may carry (item 942). Shared with execute-task's own
+ * lookup, the other one that can reach a remote.
+ */
+export function getContext(node: TaskNode, command: string) {
+  const outputs = node.config.cache?.outputs
+  return {
+    taskId: node.id,
+    command,
+    outputs: { files: outputs?.files ?? [], workspaceFiles: outputs?.workspaceFiles ?? [] },
+  }
+}

package/src/orchestrator/run-artifacts.ts

@@ -16,6 +16,7 @@
 import path from 'node:path'
 import { isGroupTask, type TaskOutcome } from '../graph/index.js'
 import { tallyOutcomes } from './tally.js'
+import type { FlakyFinding } from './failure-mode.js'
 
 export interface SummarizeArgs {
   /** Empty string → default path; otherwise the explicit file path. */
@@ -29,10 +30,12 @@ export interface SummarizeArgs {
   /** The run's verdict — the same value the CLI turns into the exit code. */
   ok: boolean
   outcomes: readonly TaskOutcome[]
+  /** What `detectFlaky` found this run; the footer's Flaky section, typed. */
+  flaky?: readonly FlakyFinding[]
 }
 
 /** One task's entry. Shared by `tasks` and `aborted` so they read alike. */
-function taskEntry(o: TaskOutcome): Record<string, unknown> {
+function taskEntry(o: TaskOutcome, flaky?: FlakyFinding): Record<string, unknown> {
   return {
     id: o.node.id,
     project: o.node.projectName,
@@ -41,8 +44,35 @@ function taskEntry(o: TaskOutcome): Record<string, unknown> {
     exitCode: o.exitCode,
     durationMs: o.durationMs,
     hash: o.hash ?? null,
+    // A task with no `cache` block executes every run by design; without
+    // the flag a consumer computing a hit rate cannot tell a miss from a
+    // task that could never hit (its `hash` is still set — dependents fold
+    // it). Present only when true, so every other row is byte-identical.
+    ...(o.node.config.cache === undefined ? { noCache: true } : {}),
+    // Present only when the run proved the task flaky: this exact key has
+    // both passed and failed on record (this run counted), or the task
+    // needed a retry. A consumer gating on `failed` can tell a break from a
+    // flake without the history; every other row is byte-identical.
+    ...(flaky !== undefined
+      ? { flaky: { passes: flaky.passes, failures: flaky.failures, attempts: flaky.attempts } }
+      : {}),
     ...(o.cpuMs !== undefined ? { cpuMs: o.cpuMs } : {}),
     ...(o.peakRssBytes !== undefined ? { peakRssBytes: o.peakRssBytes } : {}),
+    // A hit's row names what the PRODUCING execution used under its own
+    // keys, so a consumer never reads a remote worker's peak as this run's.
+    ...(o.storedCpuMs !== undefined ? { storedCpuMs: o.storedCpuMs } : {}),
+    ...(o.storedPeakRssBytes !== undefined ? { storedPeakRssBytes: o.storedPeakRssBytes } : {}),
+    // Present only when an `admit` policy held the task with a worker free.
+    ...(o.admissionHeldMs !== undefined ? { admissionHeldMs: o.admissionHeldMs } : {}),
+    // A skipped row names the failed (or aborted) task at the root of what
+    // blocked it; a fail-fast skip has none.
+    ...(o.blockedBy !== undefined ? { blockedBy: o.blockedBy } : {}),
+    // A failure vx's own `timeout` killed; absent on every other row.
+    ...(o.timedOut === true ? { timedOut: true } : {}),
+    // The sandbox's violation count; present only on a task it recorded any for.
+    ...(o.sandboxViolations !== undefined ? { sandboxViolations: o.sandboxViolations } : {}),
+    // A persistent task that never became ready, and why.
+    ...(o.notReady !== undefined ? { notReady: o.notReady } : {}),
     // hrtime spans are bigints → emit as strings so JSON.parse on
     // the consumer side doesn't truncate the ns precision.
     ...(o.wallclockStartNs !== undefined ? { wallclockStartNs: String(o.wallclockStartNs) } : {}),
@@ -65,6 +95,7 @@ export async function writeRunSummary(args: SummarizeArgs): Promise<string> {
   // exit — the terminal has said so in its Aborted section all along, and a
   // parser must not be told less than a human is.
   const aborted = args.outcomes.filter((o) => !isGroupTask(o.node) && o.status === 'aborted')
+  const flakyById = new Map((args.flaky ?? []).map((f) => [f.taskId, f]))
   const payload = {
     runId: args.runId,
     // The run-level verdict, first: a consumer gating on this artifact must
@@ -74,8 +105,8 @@ export async function writeRunSummary(args: SummarizeArgs): Promise<string> {
     startedAt: new Date(args.startedAtMs).toISOString(),
     endedAt: new Date(args.endedAtMs).toISOString(),
     totalMs: args.totalMs,
-    tasks: counted.map(taskEntry),
-    aborted: aborted.map(taskEntry),
+    tasks: counted.map((o) => taskEntry(o, flakyById.get(o.node.id))),
+    aborted: aborted.map((o) => taskEntry(o)),
     // The FULL list: `tallyOutcomes` applies the same group/aborted
     // exclusions internally, so every counted bucket is unchanged by passing
     // it — but `summary.aborted` is only non-zero if it sees them.

package/src/orchestrator/run-context.ts

@@ -11,7 +11,7 @@
 import fs from 'node:fs'
 import os from 'node:os'
 import path from 'node:path'
-import { ulid, xxh3hex } from '../util/index.js'
+import { executablePath, ulid, xxh3hex } from '../util/index.js'
 
 export interface GitContext {
   /** `git rev-parse HEAD`, or null outside a repo / on failure. */
@@ -65,7 +65,15 @@ export function captureGitContext(
   }
   try {
     const proc = Bun.spawnSync({
-      cmd: ['git', '-C', workspaceRoot, 'rev-parse', 'HEAD', '--abbrev-ref', 'HEAD'],
+      cmd: [
+        executablePath('git'),
+        '-C',
+        workspaceRoot,
+        'rev-parse',
+        'HEAD',
+        '--abbrev-ref',
+        'HEAD',
+      ],
       stdout: 'pipe',
       stderr: 'ignore',
     })
@@ -81,7 +89,7 @@ export function captureGitContext(
       // defaultBranch`) means it is at least never mistaken for trunk, but
       // every PR collapses into one scope — so one PR's timings feed another's
       // baseline, cross-branch regression detection counts them as a single
-      // branch, and the dashboard's branch column reads "HEAD".
+      // branch, and a consumer grouping runs by branch reads "HEAD".
       if (br && br !== 'HEAD') branch = br
     }
   } catch {
@@ -205,7 +213,14 @@ export function captureDefaultBranch(
 
   try {
     const proc = Bun.spawnSync({
-      cmd: ['git', '-C', workspaceRoot, 'symbolic-ref', '--short', 'refs/remotes/origin/HEAD'],
+      cmd: [
+        executablePath('git'),
+        '-C',
+        workspaceRoot,
+        'symbolic-ref',
+        '--short',
+        'refs/remotes/origin/HEAD',
+      ],
       stdout: 'pipe',
       stderr: 'ignore',
     })
@@ -310,7 +325,7 @@ export function captureWorkspaceIdentity(workspaceRoot: string): WorkspaceIdenti
       // rewrites, so two developers mirroring the same repo through
       // different proxies would derive different workspace ids. The raw
       // configured URL is the identity.
-      cmd: ['git', '-C', workspaceRoot, 'config', '--get', 'remote.origin.url'],
+      cmd: [executablePath('git'), '-C', workspaceRoot, 'config', '--get', 'remote.origin.url'],
       stdout: 'pipe',
       stderr: 'ignore',
     })