@vzn/vx 0.0.638 → 0.0.640

package/src/orchestrator/prepare.ts

@@ -24,13 +24,14 @@ import {
   putRunEntry,
   writesRunStore,
   type PluginCache,
-  type PluginCacheRemote,
+  type ResolvedRemote,
   ScopedCache,
   applyGitEnumeration,
   gitPathspecs,
   MAX_SCOPED_PATHSPECS,
   startGitEnumeration,
   lazyGitEnumeration,
+  measureFsClockLag,
 } from '../cache/index.js'
 import {
   buildPackageGraph,
@@ -50,6 +51,7 @@ import {
   expandRequested,
   type GraphRules,
   isGroupTask,
+  refuseFiltersThroughGroups,
   type TaskNode,
   undeclaredDepsError,
   unresolvedRequests,
@@ -61,6 +63,7 @@ import {
   hasHook,
   resolveCache,
   teardownPlugins,
+  type ScheduleRun,
 } from './plugin-host.js'
 import {
   discoverProjects,
@@ -78,11 +81,11 @@ import { applyGraphStage } from './graph-stage.js'
 import { keyExcludedDependencies } from './excluded-keys.js'
 import { FingerprintWatch } from './fingerprint-watch.js'
 import type { VxPlugin } from './plugin.js'
+import { askIgnoredOutputs, gatedOutputs } from './stable-keys.js'
 import { currentBranch } from './run-context.js'
 import { createHashCache, type HashCache } from './task-hash.js'
-import type { Logger } from './logger.js'
+import { type Logger, warnOf } from '../report/index.js'
 import type { RunOptions } from './options.js'
-import { refuseFiltersThroughGroups } from './upstream.js'
 import { runRules } from '../settings.js'
 
 export interface PreparedRun {
@@ -113,9 +116,10 @@ export interface PreparedRun {
    * merged over the structural baseline by the scheduler). Empty when no
    * plugin declares the stage. Asked only by a run with a task to start:
    * the stage reads history (148 ms at 8,002 tasks) to order work a
-   * nothing-changed run never dispatches. Runs the stage once.
+   * nothing-changed run never dispatches. Runs the stage once: `run`
+   * times each plugin's call and hears its later `ctx.refine` sets.
    */
-  priorities: () => Promise<ReadonlyMap<string, number>>
+  priorities: (run?: ScheduleRun) => Promise<ReadonlyMap<string, number>>
   /**
    * The run's branch (`run.branch`, else git's, else the CI provider's):
    * what its history records and the timings a forecast reads first.
@@ -241,9 +245,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
     (reused !== undefined ? gitOfDiscovery(reused.projects) : undefined) ??
     lazyGitEnumeration(workspaceRoot)
   if (wholeTree) void git.start()
-  const { workspaceConfig, plugins } = await loadWorkspacePlugins(workspaceRoot, (m) =>
-    log.status(m),
-  )
+  const { workspaceConfig, plugins } = await loadWorkspacePlugins(workspaceRoot, warnOf(log))
   mark('workspace config')
   let learnedBranch: string | null | undefined
   const branch = (): string | null =>
@@ -257,20 +259,16 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
   const cacheDir = options.cacheDir
     ? path.resolve(options.cwd, options.cacheDir)
     : resolveCacheDir(workspaceRoot, workspaceConfig)
+  // Before any racy window is judged: a file system behind this process's
+  // clock dated a mid-command edit before the command (fs-clock.ts).
+  measureFsClockLag(cacheDir)
   // Asked now, read when the cache opens: a repository with no remote is
   // named by its root commit, a git spawn the discovery below overlaps.
   const storeRoot = options.cacheDir ? null : resolveStoreRoot(workspaceRoot, workspaceConfig)
   const projectMetas =
     reused !== undefined
       ? reused.projects
-      : await discoverProjects(
-          workspaceRoot,
-          readLock,
-          plugins,
-          cacheDir,
-          (m) => log.status(m),
-          git,
-        )
+      : await discoverProjects(workspaceRoot, readLock, plugins, cacheDir, warnOf(log), git)
   mark('discover projects')
 
   // SCOPED config loading: configs are programs, and evaluating 1090
@@ -368,7 +366,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
     projectMetas,
     packageGraph,
     configs: readLock,
-    warn: (m: string) => log.status(m),
+    warn: warnOf(log),
   }
   let loaded: LoadedProjects
   try {
@@ -464,21 +462,22 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
   // winning prevents double-wrapping when the workspace also declares a
   // cache plugin.
   let cache: CacheLayer
-  const remotes: PluginCacheRemote[] = []
+  const remotes: ResolvedRemote[] = []
   if (options.remoteCache) {
-    const { remoteCache } = options
-    remotes.push({ remote: remoteCache, read: policy.remoteRead, write: policy.remoteWrite })
+    remotes.push({
+      remote: options.remoteCache,
+      read: policy.remoteRead,
+      write: policy.remoteWrite,
+      onError: (err) => warnOf(log)(`[vx] remote cache: ${err.message}`),
+    })
   }
   try {
     cache = options.remoteCache
-      ? new LayeredCache(localCache, options.remoteCache, {
-          policy,
-          onRemoteError: (err) => log.status(`[vx] remote cache: ${err.message}`),
-        })
+      ? new LayeredCache(localCache, remotes)
       : await resolveCache(plugins, {
           workspaceRoot,
           cacheDir,
-          warn: (m) => log.status(m),
+          warn: warnOf(log),
           scope: cacheScope,
           localCache,
           policy,
@@ -489,7 +488,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
     // refused by name; the local handle opened above must not leak with it,
     // nor what another plugin's factory opened: the factories have run
     // (C-4, item 1029's contract).
-    await teardownPlugins(plugins, (m) => log.status(m))
+    await teardownPlugins(plugins, warnOf(log))
     localCache.close()
     throw err
   }
@@ -581,7 +580,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
         workspaceRoot,
         cacheDir,
         projectMetas,
-        warn: (m) => log.status(m),
+        warn: warnOf(log),
         rules: graphRules,
       })
     // `--affected` selects over the FINAL graph: an edge or an input a
@@ -675,14 +674,19 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
     let usesWorkspaceInputs = false
     for (const n of nodes.values()) {
       graphDirs.add(n.projectDir)
-      if ((n.config.cache?.inputs.workspaceFiles?.length ?? 0) > 0) usesWorkspaceInputs = true
+      if ((n.config.inputs?.workspaceFiles?.length ?? 0) > 0) usesWorkspaceInputs = true
     }
     const projectDirs = [...graphDirs]
-    const enumeration = await (git.started ??
+    const enumerating =
+      git.started ??
       startGitEnumeration(
         workspaceRoot,
         gitPathspecs(workspaceRoot, projectDirs, usesWorkspaceInputs),
-      ))
+      )
+    // While git walks: the gate's pass over every task cost 2-8 ms at
+    // 1,000 projects when it ran before the spawn.
+    const settleIgnored = askIgnoredOutputs(gatedOutputs(nodes, workspaceRoot), workspaceRoot)
+    const enumeration = await enumerating
     await applyGitEnumeration(
       enumeration,
       workspaceRoot,
@@ -691,6 +695,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
       usesWorkspaceInputs,
       localCache,
     )
+    await settleIgnored(gitFilesCache)
     mark('git enumeration')
     if (hasGraphHook && !hooksFirst) await graphStage()
     // Args after `--` go to requested commands only; with none among them
@@ -706,7 +711,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
       }
     }
     if (hasHook(plugins, 'key')) {
-      await applyKeyHooks(plugins, nodes, { workspaceRoot, cacheDir, warn: (m) => log.status(m) })
+      await applyKeyHooks(plugins, nodes, { workspaceRoot, cacheDir, warn: warnOf(log) })
     }
     refuseFiltersThroughGroups(nodes)
     const exclude = options.excludeDependencies
@@ -744,7 +749,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
       return own
     }
     let scheduled: Promise<ReadonlyMap<string, number>> | undefined
-    const priorities = (): Promise<ReadonlyMap<string, number>> =>
+    const priorities = (run?: ScheduleRun): Promise<ReadonlyMap<string, number>> =>
       (scheduled ??= hasHook(plugins, 'schedule')
         ? applyScheduleHooks(
             plugins,
@@ -752,11 +757,12 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
             {
               workspaceRoot,
               cacheDir,
-              warn: (m) => log.status(m),
+              warn: warnOf(log),
               localCache,
               branch: branch(),
             },
             cacheFor,
+            run,
           )
         : Promise.resolve(new Map()))
 
@@ -790,7 +796,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
       empty: nodes.size === 0 ? 'empty-graph' : null,
     }
   } catch (err) {
-    await teardownPlugins(plugins, (m) => log.status(m))
+    await teardownPlugins(plugins, warnOf(log))
     cache.close()
     throw err
   }

package/src/orchestrator/projects.ts

@@ -258,7 +258,7 @@ export async function loadProjects(args: LoadProjectsArgs): Promise<LoadedProjec
   // Turbo spells the root package `//` (`dependsOn: ['//#lint']`), as vx
   // names the root project unless something names it; a named root's
   // spelling is rewritten to its name once, here, for the closure, the
-  // graph and `cache.inputs.tasks` alike. With no root project it stays,
+  // graph and `inputs.tasks` alike. With no root project it stays,
   // and the graph's refusal says so.
   const root = path.resolve(workspaceRoot)
   const rootName = args.projectMetas.find((m) => path.resolve(m.dir) === root)?.name
@@ -357,10 +357,7 @@ export async function loadProjects(args: LoadProjectsArgs): Promise<LoadedProjec
  */
 const DEFAULT_BUILD: TaskConfig = Object.freeze({
   dependsOn: Object.freeze(['^build']),
-  cache: Object.freeze({
-    inputs: Object.freeze({ files: Object.freeze(['**']) }),
-    outputs: Object.freeze({ files: Object.freeze([]) }),
-  }),
+  inputs: Object.freeze({ files: Object.freeze(['**']) }),
 })
 
 /** The default `build`, which a menu of the tasks a user declared leaves out. */
@@ -374,7 +371,7 @@ function withDefaultBuild(config: ProjectConfig): ProjectConfig {
 }
 
 /**
- * `config` with Turbo's `//#task` (in `dependsOn` and `cache.inputs.tasks`)
+ * `config` with Turbo's `//#task` (in `dependsOn` and `inputs.tasks`)
  * spelled `<root>#task`; the same object when it holds none. A copy, not
  * an edit: a loaded config may be a cached or a frozen one.
  */
@@ -386,14 +383,14 @@ function rootSpelled(config: ProjectConfig, rootName: string): ProjectConfig {
   let tasks: NonNullable<ProjectConfig['tasks']> | undefined
   for (const [name, task] of Object.entries(config.tasks ?? {})) {
     const dependsOn = fix(task.dependsOn)
-    const inputTasks = fix(task.cache?.inputs?.tasks)
-    if (dependsOn === task.dependsOn && inputTasks === task.cache?.inputs?.tasks) continue
+    const inputTasks = fix(task.inputs?.tasks)
+    if (dependsOn === task.dependsOn && inputTasks === task.inputs?.tasks) continue
     tasks ??= { ...config.tasks }
     tasks[name] = {
       ...task,
       ...(dependsOn !== undefined ? { dependsOn: [...dependsOn] } : {}),
-      ...(task.cache?.inputs !== undefined && inputTasks !== undefined
-        ? { cache: { ...task.cache, inputs: { ...task.cache.inputs, tasks: [...inputTasks] } } }
+      ...(task.inputs !== undefined && inputTasks !== undefined
+        ? { inputs: { ...task.inputs, tasks: [...inputTasks] } }
         : {}),
     }
   }

package/src/orchestrator/remote-prefetch.ts

@@ -41,6 +41,11 @@ export interface PrefetchArgs {
   concurrency: number
   /** When false (remote reads disabled), no prefetch fires. */
   remoteRead: boolean
+  /**
+   * With reads off and writes on (`--force`), the keys go to
+   * `probeBeforeUpload` instead, so a key a remote holds is not re-sent.
+   */
+  remoteWrite: boolean
 }
 
 /**
@@ -55,7 +60,15 @@ export interface PrefetchArgs {
  * is always correct.
  */
 export function startRemotePrefetch(args: PrefetchArgs): Promise<void> {
-  if (!args.remoteRead) return Promise.resolve()
+  if (!args.remoteRead) {
+    if (!args.remoteWrite || args.cache.probeBeforeUpload === undefined) return Promise.resolve()
+    const hashes = deriveStableKeys(args).then((keys) => [...new Set(keys.map((k) => k.hash))])
+    args.cache.probeBeforeUpload(hashes)
+    return hashes.then(
+      () => {},
+      () => {},
+    )
+  }
   // Detached from EXECUTION: the caller does NOT await this before
   // scheduling, so prefetch network latency overlaps the run. But the
   // caller DOES await the returned handle before closing the cache, so

package/src/orchestrator/run-artifacts.ts

@@ -1,182 +1,10 @@
-// Optional per-run output artifacts:
-//
-//   --summarize[=<path>] → JSON describing every task's outcome
-//                          (id, status, exit code, durations, hash,
-//                          cpu_ms, peak_rss_bytes, hrtime spans).
-//                          Default path: <cacheDir>/runs/<run_id>.json.
-//
-//   --profile[=<path>]   → Chrome-trace JSON of the run. Each task is
-//                          a single complete event (`ph: 'X'`) with
-//                          ts / dur in microseconds derived from the
-//                          hrtime spans the runner already captures.
-//                          Default path: profile.json (cwd-relative).
-//
-// Both are no-ops if the corresponding RunOptions field is undefined.
+// `--profile[=<path>]`: Chrome-trace JSON of the run. Each task is a single
+// complete event (`ph: 'X'`) with ts / dur in microseconds derived from the
+// hrtime spans the runner already captures. Default path: profile.json
+// (cwd-relative).
 
 import path from 'node:path'
-import { isGroupTask, type TaskOutcome } from '../graph/index.js'
-import { tallyOutcomes, type Tally } from './tally.js'
-import type { FlakyFinding } from './failure-mode.js'
-import { isCacheHit } from './telemetry.js'
-import { ranNoCache } from './events.js'
-
-export interface SummarizeArgs {
-  /** Empty string → default path; otherwise the explicit file path. */
-  target: string
-  cacheDir: string
-  cwd: string
-  runId: string
-  startedAtMs: number
-  endedAtMs: number
-  totalMs: number
-  /** The run's verdict — the same value the CLI turns into the exit code. */
-  ok: boolean
-  /** The process's exit code where it is not `ok ? 0 : 1`: a stopping signal's. */
-  exitCode?: number
-  outcomes: readonly TaskOutcome[]
-  /** What `detectFlaky` found this run; each row's flaky note, typed. */
-  flaky?: readonly FlakyFinding[]
-}
-
-/** One task of `--summarize`: the wire, stated by `schemas/summary.json`. */
-export interface SummaryTaskJson {
-  id: string
-  project: string
-  task: string
-  status: TaskOutcome['status']
-  exitCode: number
-  durationMs: number
-  hash: string | null
-  restored?: boolean
-  noCache?: true
-  flaky?: { passes: number; failures: number; attempts: number }
-  cpuMs?: number
-  peakRssBytes?: number
-  storedDurationMs?: number
-  storedCpuMs?: number
-  storedPeakRssBytes?: number
-  admissionHeldMs?: number
-  blockedBy?: string
-  timedOut?: true
-  sandboxViolations?: number
-  notReady?: 'timeout' | 'exited' | 'spawn'
-  wallclockStartNs?: string
-  wallclockEndNs?: string
-}
-
-/** The `--summarize` document. */
-export interface RunSummaryJson {
-  runId: string
-  ok: boolean
-  exitCode: number
-  startedAt: string
-  endedAt: string
-  totalMs: number
-  /** What the hits skipped: the sum of their `storedDurationMs`. */
-  savedMs: number
-  tasks: SummaryTaskJson[]
-  aborted: SummaryTaskJson[]
-  summary: Tally
-}
-
-/** One task's entry. Shared by `tasks` and `aborted` so they read alike. */
-function taskEntry(o: TaskOutcome, flaky?: FlakyFinding): SummaryTaskJson {
-  return {
-    id: o.node.id,
-    project: o.node.projectName,
-    task: o.node.taskName,
-    status: o.status,
-    exitCode: o.exitCode,
-    durationMs: o.durationMs,
-    hash: o.hash ?? null,
-    // On a hit: whether it restored outputs or found them up to date, as
-    // `summary`'s restoredLocal / restoredRemote / upToDate count it.
-    ...(isCacheHit(o.status) ? { restored: o.restored === true } : {}),
-    // A task with no `cache` block executes every run by design; without
-    // the flag a consumer computing a hit rate cannot tell a miss from a
-    // task that could never hit (its `hash` is still set — dependents fold
-    // it). Present only when true, so every other row is byte-identical.
-    ...(ranNoCache(o) ? { noCache: true } : {}),
-    // Present only when the run proved the task flaky: this exact key
-    // failed after it had passed (this run counted), or the task
-    // needed a retry. A consumer gating on `failed` can tell a break from a
-    // flake without the history; every other row is byte-identical.
-    ...(flaky !== undefined
-      ? { flaky: { passes: flaky.passes, failures: flaky.failures, attempts: flaky.attempts } }
-      : {}),
-    ...(o.cpuMs !== undefined ? { cpuMs: o.cpuMs } : {}),
-    ...(o.peakRssBytes !== undefined ? { peakRssBytes: o.peakRssBytes } : {}),
-    // A hit's row names what the PRODUCING execution used under its own
-    // keys, so a consumer never reads a remote worker's peak as this run's.
-    ...(o.storedDurationMs !== undefined ? { storedDurationMs: o.storedDurationMs } : {}),
-    ...(o.storedCpuMs !== undefined ? { storedCpuMs: o.storedCpuMs } : {}),
-    ...(o.storedPeakRssBytes !== undefined ? { storedPeakRssBytes: o.storedPeakRssBytes } : {}),
-    // Present only when an `admit` policy held the task with a worker free.
-    ...(o.admissionHeldMs !== undefined ? { admissionHeldMs: o.admissionHeldMs } : {}),
-    // A skipped row names the failed (or aborted) task at the root of what
-    // blocked it; a fail-fast skip has none.
-    ...(o.blockedBy !== undefined ? { blockedBy: o.blockedBy } : {}),
-    // A failure vx's own `timeout` killed; absent on every other row.
-    ...(o.timedOut === true ? { timedOut: true } : {}),
-    // The sandbox's violation count; present only on a task it recorded any for.
-    ...(o.sandboxViolations !== undefined ? { sandboxViolations: o.sandboxViolations } : {}),
-    // A persistent task that never became ready, and why.
-    ...(o.notReady !== undefined ? { notReady: o.notReady } : {}),
-    // hrtime spans are bigints → emit as strings so JSON.parse on
-    // the consumer side doesn't truncate the ns precision.
-    ...(o.wallclockStartNs !== undefined ? { wallclockStartNs: String(o.wallclockStartNs) } : {}),
-    ...(o.wallclockEndNs !== undefined ? { wallclockEndNs: String(o.wallclockEndNs) } : {}),
-  }
-}
-
-export async function writeRunSummary(args: SummarizeArgs): Promise<string> {
-  const outPath =
-    args.target === ''
-      ? path.join(args.cacheDir, 'runs', `${args.runId}.json`)
-      : path.resolve(args.cwd, args.target)
-  await Bun.write(outPath, JSON.stringify(runSummaryJson(args), null, 2))
-  return outPath
-}
-
-/** The `--summarize` document, which `vx run --format json` prints. */
-export function runSummaryJson(
-  args: Omit<SummarizeArgs, 'target' | 'cacheDir' | 'cwd'>,
-): RunSummaryJson {
-  // `tasks` and `summary` must describe the same population, or one artifact
-  // contradicts itself (a group task listed in `tasks` but absent from
-  // `summary.total`). `tallyOutcomes` owns the rule; mirror its filter here.
-  const counted = args.outcomes.filter((o) => !isGroupTask(o.node) && o.status !== 'aborted')
-  // A task killed by a shutdown signal is in no bucket and no total, yet it
-  // makes the run red. Listing it separately keeps `tasks.length ===
-  // summary.total` while leaving the artifact able to explain a non-zero
-  // exit — the terminal has said so in its Aborted section all along, and a
-  // parser must not be told less than a human is.
-  const aborted = args.outcomes.filter((o) => !isGroupTask(o.node) && o.status === 'aborted')
-  const flakyById = new Map((args.flaky ?? []).map((f) => [f.taskId, f]))
-  const exitCode = args.exitCode ?? (args.ok ? 0 : 1)
-  const payload: RunSummaryJson = {
-    runId: args.runId,
-    // The run-level verdict, first: a consumer gating on this artifact must
-    // not have to re-derive it by re-implementing the bucket rules.
-    ok: exitCode === 0,
-    exitCode,
-    startedAt: new Date(args.startedAtMs).toISOString(),
-    endedAt: new Date(args.endedAtMs).toISOString(),
-    totalMs: args.totalMs,
-    // The summary row's `saved`, by the same rule: hits only, groups aside.
-    savedMs: counted.reduce(
-      (ms, o) => (isCacheHit(o.status) ? ms + (o.storedDurationMs ?? 0) : ms),
-      0,
-    ),
-    tasks: counted.map((o) => taskEntry(o, flakyById.get(o.node.id))),
-    aborted: aborted.map((o) => taskEntry(o)),
-    // The FULL list: `tallyOutcomes` applies the same group/aborted
-    // exclusions internally, so every counted bucket is unchanged by passing
-    // it — but `summary.aborted` is only non-zero if it sees them.
-    summary: tallyOutcomes(args.outcomes),
-  }
-  return payload
-}
+import type { TaskOutcome } from '../graph/index.js'
 
 export interface ProfileArgs {
   target: string

package/src/orchestrator/run-context.ts

@@ -15,6 +15,8 @@ import os from 'node:os'
 import path from 'node:path'
 import { executablePath, gitEnv, relPosix, ulid, xxh3hex } from '../util/index.js'
 import { resolveSetting } from '../settings.js'
+import { VERSION } from '../version.js'
+import type { RunContextRecord } from './telemetry.js'
 
 export interface GitContext {
   /** `git rev-parse HEAD`, or null outside a repo / on failure. */
@@ -522,3 +524,56 @@ export function currentBranch(
   const set = resolveSetting('run.branch', workspace, flag, env).value as string | undefined
   return set ?? captureGitContext(workspaceRoot, null, env).branch
 }
+
+export interface TelemetryRunContextArgs {
+  runId: string
+  /** The invocation, already masked. */
+  command: string
+  requestedTasks: readonly string[]
+  cachePolicy: string
+  concurrency: number
+  flow: RunContextRecord['flow']
+  git: GitContext
+  ci: CiContext
+  host: HostContext
+  workspaceRoot: string
+  /** `remote.origin.url` as the run's enumeration read it; see `captureWorkspaceIdentity`. */
+  originUrl: string | null | undefined
+}
+
+/**
+ * The run-context record telemetry sinks receive: the invocation header's
+ * git/CI/host data shaped as the export contract. Paid only when a sink can
+ * exist: the workspace identity may write `.vx/cache/workspace-id`.
+ */
+export function telemetryRunContext(a: TelemetryRunContextArgs): RunContextRecord {
+  const ws = captureWorkspaceIdentity(a.workspaceRoot, a.originUrl)
+  const { ci } = a
+  return {
+    runId: a.runId,
+    vxVersion: VERSION,
+    command: a.command,
+    requestedTasks: [...a.requestedTasks],
+    cachePolicy: a.cachePolicy,
+    concurrency: a.concurrency,
+    flow: a.flow,
+    commitSha: a.git.commitSha,
+    branch: a.git.branch,
+    defaultBranch: captureDefaultBranch(process.env, a.workspaceRoot),
+    dirty: a.git.dirty,
+    ci: ci.ci,
+    ciProvider: ci.provider,
+    ...(ci.runUrl !== undefined ? { ciRunUrl: ci.runUrl } : {}),
+    ...(ci.change !== undefined ? { ciChange: ci.change } : {}),
+    ...(ci.pipeline !== undefined ? { ciPipeline: ci.pipeline } : {}),
+    ...(ci.job !== undefined ? { ciJob: ci.job } : {}),
+    ...(ci.attempt !== undefined ? { ciAttempt: ci.attempt } : {}),
+    host: a.host.host,
+    os: a.host.os,
+    arch: a.host.arch,
+    workspaceId: ws.id,
+    workspaceName: ws.name,
+    ...(ws.repository !== undefined ? { repository: ws.repository } : {}),
+    ...(ws.path !== undefined ? { workspacePath: ws.path } : {}),
+  }
+}

package/src/orchestrator/run-failures.ts

@@ -8,9 +8,9 @@ import { mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync
 import path from 'node:path'
 import { isGroupTask, type TaskOutcome } from '../graph/index.js'
 import { decodeOutputLog } from './output-log.js'
-import { PASS_STATUSES } from './history.js'
-import { splitTaskId } from '../util/index.js'
-import { fileLocations, fileMemo, type OutputLocation } from './path-links.js'
+import { PASS_STATUSES } from '../history/index.js'
+import { isFsRefusal, splitTaskId } from '../util/index.js'
+import { fileLocations, fileMemo, type OutputLocation } from '../report/index.js'
 
 /** Failed runs whose output is kept; run ids are UUIDv7, so names sort by age. */
 const KEPT = 50
@@ -78,14 +78,26 @@ function failuresOf(outcomes: readonly TaskOutcome[]): StoredFailure[] {
 /**
  * Write the run's failures, when any outcome carries output, and drop all
  * but the newest `KEPT` files. Atomic: a reader sees the whole file or none.
+ * A refused write (a full disk, a read-only cache dir) costs only this
+ * output: `warn` says so and the run's verdict stands.
  */
 export function writeRunFailures(
   cacheDir: string,
   runId: string,
   outcomes: readonly TaskOutcome[],
+  warn: (message: string) => void,
 ): void {
   const tasks = failuresOf(outcomes)
   if (tasks.length === 0) return
+  try {
+    keep(cacheDir, runId, tasks)
+  } catch (err) {
+    if (!isFsRefusal(err)) throw err
+    warn(`[vx] failed tasks' output not kept: ${err.message} — the verdict above stands`)
+  }
+}
+
+function keep(cacheDir: string, runId: string, tasks: TaskFailure[]): void {
   const dir = dirOf(cacheDir)
   mkdirSync(dir, { recursive: true })
   const file = path.join(dir, `${runId}.json`)

package/src/orchestrator/run-records.ts

@@ -10,7 +10,8 @@ import type { InvocationRecord, RunRecord } from '../cache/index.js'
 import { isGroupTask, type TaskOutcome } from '../graph/index.js'
 import { VERSION } from '../version.js'
 import type { CiContext, GitContext, HostContext } from './run-context.js'
-import { isCacheHit, taskTelemetryOf, type TaskTelemetry } from './telemetry.js'
+import { taskTelemetryOf, type TaskTelemetry } from './telemetry.js'
+import { isCacheHit } from '../graph/index.js'
 
 export interface RunRecordsInput {
   outcomes: readonly TaskOutcome[]
@@ -32,12 +33,13 @@ export interface RunRecordsInput {
   concurrency: number
   flow: InvocationRecord['flow']
   forwardArgs?: readonly string[] | undefined
-  tags: Record<string, string>
   git: GitContext
   ci: CiContext
   host: HostContext
   /** Build the telemetry mirror; a run with no sink allocates none. */
   withTelemetry: boolean
+  /** The run's remote failures by key, so far (`Cache.traffic`). */
+  remoteTraffic?: ReadonlyMap<string, { failures: readonly string[] }>
 }
 
 export interface RunRecords {
@@ -46,6 +48,15 @@ export interface RunRecords {
   telemetryTasks: TaskTelemetry[]
 }
 
+/** A key's remote failures as its `runs` row stores them; none when it met none. */
+function remoteErrorOf(
+  traffic: ReadonlyMap<string, { failures: readonly string[] }> | undefined,
+  hash: string | undefined,
+): string | undefined {
+  const failures = hash === undefined ? undefined : traffic?.get(hash)?.failures
+  return failures === undefined || failures.length === 0 ? undefined : failures.join('; ')
+}
+
 /**
  * Every non-group, non-aborted outcome gets a row. An outcome with NO hash
  * (a `skipped` task never probed the cache; a `persistent` one is never
@@ -71,6 +82,7 @@ export function assembleRunRecords(input: RunRecordsInput): RunRecords {
     if (o.status === 'aborted') continue
     if (input.withTelemetry) telemetryTasks.push(taskTelemetryOf(o))
     const blockedBy = o.blockedBy ?? input.tainted?.get(o.node.id)
+    const remoteError = remoteErrorOf(input.remoteTraffic, o.hash)
     runs.push({
       ...(o.hash !== undefined ? { hash: o.hash } : {}),
       project: o.node.projectName,
@@ -113,6 +125,7 @@ export function assembleRunRecords(input: RunRecordsInput): RunRecords {
       ...(o.timedOut === true ? { timedOut: true as const } : {}),
       ...(o.sandboxViolations !== undefined ? { sandboxViolations: o.sandboxViolations } : {}),
       ...(o.notReady !== undefined ? { notReady: o.notReady } : {}),
+      ...(remoteError !== undefined ? { remoteError } : {}),
     })
     if (o.status === 'failed') failedCount++
     if (o.status === 'cache-hit') hitLocalCount++
@@ -151,7 +164,26 @@ export function assembleRunRecords(input: RunRecordsInput): RunRecords {
     os: input.host.os,
     arch: input.host.arch,
     vxVersion: VERSION,
-    tags: JSON.stringify(input.tags),
   }
   return { runs, invocation, telemetryTasks }
 }
+
+/**
+ * The remote failures uploads met after the run's rows went in (the uploads
+ * drain later): each row whose text is now short of its key's is rewritten.
+ */
+export function recordLateRemoteErrors(
+  cache: {
+    readonly traffic: { readonly keys: ReadonlyMap<string, { failures: readonly string[] }> }
+    recordRemoteErrors(runId: string, errors: readonly (readonly [string, string])[]): void
+  },
+  runs: readonly RunRecord[],
+  runId: string,
+): void {
+  const late: [string, string][] = []
+  for (const r of runs) {
+    const now = remoteErrorOf(cache.traffic.keys, r.hash)
+    if (now !== undefined && now !== r.remoteError) late.push([r.hash!, now])
+  }
+  if (late.length > 0) cache.recordRemoteErrors(runId, late)
+}