@vzn/vx 0.0.639 → 0.0.641

package/src/orchestrator/prepare.ts

@@ -31,6 +31,7 @@ import {
   MAX_SCOPED_PATHSPECS,
   startGitEnumeration,
   lazyGitEnumeration,
+  measureFsClockLag,
 } from '../cache/index.js'
 import {
   buildPackageGraph,
@@ -50,6 +51,7 @@ import {
   expandRequested,
   type GraphRules,
   isGroupTask,
+  refuseFiltersThroughGroups,
   type TaskNode,
   undeclaredDepsError,
   unresolvedRequests,
@@ -61,6 +63,7 @@ import {
   hasHook,
   resolveCache,
   teardownPlugins,
+  type ScheduleRun,
 } from './plugin-host.js'
 import {
   discoverProjects,
@@ -83,7 +86,6 @@ import { currentBranch } from './run-context.js'
 import { createHashCache, type HashCache } from './task-hash.js'
 import { type Logger, warnOf } from '../report/index.js'
 import type { RunOptions } from './options.js'
-import { refuseFiltersThroughGroups } from './upstream.js'
 import { runRules } from '../settings.js'
 
 export interface PreparedRun {
@@ -114,9 +116,10 @@ export interface PreparedRun {
    * merged over the structural baseline by the scheduler). Empty when no
    * plugin declares the stage. Asked only by a run with a task to start:
    * the stage reads history (148 ms at 8,002 tasks) to order work a
-   * nothing-changed run never dispatches. Runs the stage once.
+   * nothing-changed run never dispatches. Runs the stage once: `run`
+   * times each plugin's call and hears its later `ctx.refine` sets.
    */
-  priorities: () => Promise<ReadonlyMap<string, number>>
+  priorities: (run?: ScheduleRun) => Promise<ReadonlyMap<string, number>>
   /**
    * The run's branch (`run.branch`, else git's, else the CI provider's):
    * what its history records and the timings a forecast reads first.
@@ -256,6 +259,9 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
   const cacheDir = options.cacheDir
     ? path.resolve(options.cwd, options.cacheDir)
     : resolveCacheDir(workspaceRoot, workspaceConfig)
+  // Before any racy window is judged: a file system behind this process's
+  // clock dated a mid-command edit before the command (fs-clock.ts).
+  measureFsClockLag(cacheDir)
   // Asked now, read when the cache opens: a repository with no remote is
   // named by its root commit, a git spawn the discovery below overlaps.
   const storeRoot = options.cacheDir ? null : resolveStoreRoot(workspaceRoot, workspaceConfig)
@@ -668,7 +674,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
     let usesWorkspaceInputs = false
     for (const n of nodes.values()) {
       graphDirs.add(n.projectDir)
-      if ((n.config.cache?.inputs.workspaceFiles?.length ?? 0) > 0) usesWorkspaceInputs = true
+      if ((n.config.inputs?.workspaceFiles?.length ?? 0) > 0) usesWorkspaceInputs = true
     }
     const projectDirs = [...graphDirs]
     const enumerating =
@@ -743,7 +749,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
       return own
     }
     let scheduled: Promise<ReadonlyMap<string, number>> | undefined
-    const priorities = (): Promise<ReadonlyMap<string, number>> =>
+    const priorities = (run?: ScheduleRun): Promise<ReadonlyMap<string, number>> =>
       (scheduled ??= hasHook(plugins, 'schedule')
         ? applyScheduleHooks(
             plugins,
@@ -756,6 +762,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
               branch: branch(),
             },
             cacheFor,
+            run,
           )
         : Promise.resolve(new Map()))
 

package/src/orchestrator/projects.ts

@@ -258,7 +258,7 @@ export async function loadProjects(args: LoadProjectsArgs): Promise<LoadedProjec
   // Turbo spells the root package `//` (`dependsOn: ['//#lint']`), as vx
   // names the root project unless something names it; a named root's
   // spelling is rewritten to its name once, here, for the closure, the
-  // graph and `cache.inputs.tasks` alike. With no root project it stays,
+  // graph and `inputs.tasks` alike. With no root project it stays,
   // and the graph's refusal says so.
   const root = path.resolve(workspaceRoot)
   const rootName = args.projectMetas.find((m) => path.resolve(m.dir) === root)?.name
@@ -357,10 +357,7 @@ export async function loadProjects(args: LoadProjectsArgs): Promise<LoadedProjec
  */
 const DEFAULT_BUILD: TaskConfig = Object.freeze({
   dependsOn: Object.freeze(['^build']),
-  cache: Object.freeze({
-    inputs: Object.freeze({ files: Object.freeze(['**']) }),
-    outputs: Object.freeze({ files: Object.freeze([]) }),
-  }),
+  inputs: Object.freeze({ files: Object.freeze(['**']) }),
 })
 
 /** The default `build`, which a menu of the tasks a user declared leaves out. */
@@ -374,7 +371,7 @@ function withDefaultBuild(config: ProjectConfig): ProjectConfig {
 }
 
 /**
- * `config` with Turbo's `//#task` (in `dependsOn` and `cache.inputs.tasks`)
+ * `config` with Turbo's `//#task` (in `dependsOn` and `inputs.tasks`)
  * spelled `<root>#task`; the same object when it holds none. A copy, not
  * an edit: a loaded config may be a cached or a frozen one.
  */
@@ -386,14 +383,14 @@ function rootSpelled(config: ProjectConfig, rootName: string): ProjectConfig {
   let tasks: NonNullable<ProjectConfig['tasks']> | undefined
   for (const [name, task] of Object.entries(config.tasks ?? {})) {
     const dependsOn = fix(task.dependsOn)
-    const inputTasks = fix(task.cache?.inputs?.tasks)
-    if (dependsOn === task.dependsOn && inputTasks === task.cache?.inputs?.tasks) continue
+    const inputTasks = fix(task.inputs?.tasks)
+    if (dependsOn === task.dependsOn && inputTasks === task.inputs?.tasks) continue
     tasks ??= { ...config.tasks }
     tasks[name] = {
       ...task,
       ...(dependsOn !== undefined ? { dependsOn: [...dependsOn] } : {}),
-      ...(task.cache?.inputs !== undefined && inputTasks !== undefined
-        ? { cache: { ...task.cache, inputs: { ...task.cache.inputs, tasks: [...inputTasks] } } }
+      ...(task.inputs !== undefined && inputTasks !== undefined
+        ? { inputs: { ...task.inputs, tasks: [...inputTasks] } }
         : {}),
     }
   }

package/src/orchestrator/run-report.ts

@@ -1,8 +1,14 @@
 // The report every plugin's teardown reads, and the run's own bytes kept
 // in a remote that takes its writes (owner 2026-10-15): a failed task's
-// log and the files its outputs left, an uncached task's log.
+// log and the files its outputs left (`cache.upload.failures`), and the
+// log of a task that saved no entry, on every run (M, 2026-10-16: "Always
+// keep"). Each task's bytes leave when it ends, off its path, like its
+// entry's upload (owner 2026-10-16); the run only waits for them where it
+// waits for every upload, and the footer counts them as the task's remote
+// traffic.
 
 import {
+  type RemoteTraffic,
   packRunOutputs,
   resolveOutputs,
   resolveWorkspaceOutputs,
@@ -14,17 +20,29 @@ import { relPosix } from '../util/index.js'
 import { decodeOutputLog } from './output-log.js'
 import type { RunReport, RunReportTask } from './plugin.js'
 
-export interface RunReportArgs {
+/** Stores one run entry; resolves whether a remote kept it. */
+export type KeepRunEntry = (name: string, bytes: Uint8Array) => Promise<boolean>
+
+export interface RunKeeperArgs {
   readonly runId: string
-  readonly commit: string | null
-  readonly ciRun: CiContext
-  readonly ok: boolean
   readonly workspaceRoot: string
   readonly nestedDirsByProject: ReadonlyMap<string, string[]>
-  /** Stores one run entry; undefined when no remote takes the run's writes. */
-  readonly keep: ((name: string, bytes: Uint8Array) => Promise<boolean>) | undefined
+  readonly keep: KeepRunEntry
+  /** Whether a failed task's log and outputs are kept (`cache.upload.failures`). */
+  readonly failures: boolean
+  /** The footer's remote traffic: each kept task counts under its id. */
+  readonly traffic: RemoteTraffic
+}
+
+/** Whether a run on CI or off it keeps failures, by `cache.upload.failures` (default `'ci'`). */
+export function keepsFailures(where: unknown, ci: boolean): boolean {
+  const at = where ?? 'ci'
+  return at === 'always' || (at === 'ci') === ci
 }
 
+/** Concurrent run-entry writes; more queue. */
+const KEEP_CONCURRENCY = 8
+
 /** The files `node`'s declared outputs match on disk now, workspace-relative and sorted. */
 async function outputsLeft(
   node: TaskNode,
@@ -40,52 +58,133 @@ async function outputsLeft(
   return [...new Set([...own, ...ws].map((p) => relPosix(workspaceRoot, p)))].sort()
 }
 
-export async function keepRunBytes(
-  final: readonly TaskOutcome[],
-  args: RunReportArgs,
-): Promise<RunReport> {
-  const { keep, runId, workspaceRoot } = args
-  const tasks: RunReportTask[] = []
-  const work: (() => Promise<void>)[] = []
-  for (const o of final) {
-    if (isGroupTask(o.node)) continue
-    const task: { -readonly [K in keyof RunReportTask]: RunReportTask[K] } = {
-      id: o.node.id,
-      status: o.status,
-      durationMs: Math.round(o.durationMs),
-      ...(o.hash !== undefined ? { key: o.hash } : {}),
+/** Keeps each task's run bytes as it ends, and names them in the run's report. */
+export class RunKeeper {
+  private readonly kept = new Map<string, { log?: string; outputs?: string }>()
+  private readonly queue: (() => Promise<void>)[] = []
+  private readonly pending = new Set<Promise<void>>()
+  private active = 0
+  private stopped = false
+
+  constructor(private readonly args: RunKeeperArgs) {}
+
+  /** A task ended: queue what of it the run keeps. */
+  taskDone(o: TaskOutcome): void {
+    if (this.stopped || isGroupTask(o.node)) return
+    const { runId, workspaceRoot } = this.args
+    const id = o.node.id
+    const slot = (): { log?: string; outputs?: string } => {
+      let s = this.kept.get(id)
+      if (s === undefined) this.kept.set(id, (s = {}))
+      return s
     }
-    tasks.push(task)
-    if (keep === undefined) continue
     const runLog = o.runLog
     if (runLog !== undefined) {
-      work.push(async () => {
-        const text = decodeOutputLog(runLog)
-          .map((c) => c.text)
-          .join('')
-        if (text.length === 0) return
-        const name = runEntryName(runId, o.node.id, 'log')
-        if (await keep(name, new TextEncoder().encode(text))) task.log = name
+      const text = decodeOutputLog(runLog)
+        .map((c) => c.text)
+        .join('')
+      if (text.length > 0) {
+        const name = runEntryName(runId, id, 'log')
+        const bytes = new TextEncoder().encode(text)
+        this.enqueue(id, async () => {
+          if (!(await this.args.keep(name, bytes))) return false
+          slot().log = name
+          return true
+        })
+      }
+    }
+    if (this.args.failures && o.status === 'failed' && o.where === undefined) {
+      // Read now: a later task may write where this one's outputs lie.
+      const packed = outputsLeft(
+        o.node,
+        workspaceRoot,
+        this.args.nestedDirsByProject.get(o.node.projectName) ?? [],
+      ).then((paths) => packRunOutputs(workspaceRoot, paths))
+      const name = runEntryName(runId, id, 'outputs')
+      this.enqueue(id, async () => {
+        const bytes = await packed
+        if (bytes === null || !(await this.args.keep(name, bytes))) return false
+        slot().outputs = name
+        return true
       })
     }
-    if (o.status === 'failed' && o.where === undefined) {
-      work.push(async () => {
-        const bytes = await packRunOutputs(
-          workspaceRoot,
-          await outputsLeft(
-            o.node,
-            workspaceRoot,
-            args.nestedDirsByProject.get(o.node.projectName) ?? [],
-          ),
-        )
-        if (bytes === null) return
-        const name = runEntryName(runId, o.node.id, 'outputs')
-        if (await keep(name, bytes)) task.outputs = name
+  }
+
+  /** Resolves once every queued write has settled, or the run was stopped. */
+  async settled(): Promise<void> {
+    while (!this.stopped && this.pending.size > 0) await Promise.all(this.pending)
+  }
+
+  /** A stopped run starts no more writes and stops waiting on those in flight. */
+  stop(): void {
+    this.stopped = true
+    this.queue.length = 0
+  }
+
+  /** What was kept of each task, by id, for `reportOf`. */
+  keptOf(id: string): { log?: string; outputs?: string } | undefined {
+    return this.kept.get(id)
+  }
+
+  /** Queue `job`, which writes `id`'s bytes; a write that lands counts as `id`'s upload. */
+  private enqueue(id: string, job: () => Promise<boolean>): void {
+    let done!: () => void
+    const p = new Promise<void>((resolve) => (done = resolve))
+    this.pending.add(p)
+    const { traffic } = this.args
+    let key = traffic.keys.get(id)
+    if (key === undefined)
+      traffic.keys.set(id, (key = { uploading: 0, landed: false, failures: [] }))
+    const counted = key
+    counted.uploading++
+    traffic.inFlight++
+    this.queue.push(async () => {
+      try {
+        if (!this.stopped && (await job())) counted.landed = true
+      } catch {
+        // a record, never the run's verdict
+      } finally {
+        counted.uploading--
+        traffic.inFlight--
+        if (counted.uploading === 0 && !counted.landed && counted.failures.length === 0)
+          traffic.keys.delete(id)
+        this.pending.delete(p)
+        done()
+      }
+    })
+    this.pump()
+  }
+
+  private pump(): void {
+    while (this.active < KEEP_CONCURRENCY && this.queue.length > 0) {
+      const job = this.queue.shift()!
+      this.active++
+      void job().finally(() => {
+        this.active--
+        this.pump()
       })
     }
   }
-  for (let i = 0; i < work.length; i += 8) {
-    await Promise.all(work.slice(i, i + 8).map((w) => w()))
+}
+
+/** The report: every task, with the names of what `keeper` kept of it. */
+export function reportOf(
+  final: readonly TaskOutcome[],
+  run: { runId: string; commit: string | null; ciRun: CiContext; ok: boolean },
+  keeper: RunKeeper | undefined,
+): RunReport {
+  const tasks: RunReportTask[] = []
+  for (const o of final) {
+    if (isGroupTask(o.node)) continue
+    const kept = keeper?.keptOf(o.node.id)
+    tasks.push({
+      id: o.node.id,
+      status: o.status,
+      durationMs: Math.round(o.durationMs),
+      ...(o.hash !== undefined ? { key: o.hash } : {}),
+      ...(kept?.log !== undefined ? { log: kept.log } : {}),
+      ...(kept?.outputs !== undefined ? { outputs: kept.outputs } : {}),
+    })
   }
-  return { runId, commit: args.commit, ciRun: args.ciRun, ok: args.ok, tasks }
+  return { runId: run.runId, commit: run.commit, ciRun: run.ciRun, ok: run.ok, tasks }
 }

package/src/orchestrator/run-steps.ts

@@ -0,0 +1,57 @@
+// The steps a run did not wait on (`vx:prefetch`), for `vx last` (M,
+// 2026-10-16): they never show in a run's output. One file per run that
+// had one, `<cacheDir>/steps/<runId>.json`, the newest 50 kept, as
+// run-failures.ts keeps failed output: files, not cache.db, so the index
+// layout stays as it was. A run with none writes nothing.
+
+import { mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'
+import path from 'node:path'
+import { isFsRefusal } from '../util/index.js'
+
+/** Runs whose background steps are kept; run ids are UUIDv7, so names sort by age. */
+const KEPT = 50
+
+/** One of vx's own steps that ran beside the tasks. */
+export interface RunStep {
+  id: string
+  durationMs: number
+  ok: boolean
+}
+
+const dirOf = (cacheDir: string): string => path.join(cacheDir, 'steps')
+
+/** Keep `steps` for `runId`; a refused write costs only them. */
+export function writeRunSteps(
+  cacheDir: string,
+  runId: string,
+  steps: readonly RunStep[],
+  warn: (message: string) => void,
+): void {
+  if (steps.length === 0) return
+  try {
+    const dir = dirOf(cacheDir)
+    mkdirSync(dir, { recursive: true })
+    const file = path.join(dir, `${runId}.json`)
+    const tmp = `${file}.${process.pid}.tmp`
+    writeFileSync(tmp, JSON.stringify(steps))
+    renameSync(tmp, file)
+    const names = readdirSync(dir)
+      .filter((n) => n.endsWith('.json'))
+      .sort()
+    for (const n of names.slice(0, -KEPT)) rmSync(path.join(dir, n), { force: true })
+  } catch (err) {
+    if (!isFsRefusal(err)) throw err
+    warn(`[vx] background steps not kept: ${err.message} — the verdict above stands`)
+  }
+}
+
+/** The background steps `runId` kept; none when it had none or they are gone. */
+export function runSteps(cacheDir: string, runId: string): RunStep[] {
+  try {
+    return JSON.parse(
+      readFileSync(path.join(dirOf(cacheDir), `${runId}.json`), 'utf8'),
+    ) as RunStep[]
+  } catch {
+    return []
+  }
+}