@vzn/vx 0.0.637 → 0.0.639

package/src/orchestrator/run-records.ts

@@ -10,7 +10,8 @@ import type { InvocationRecord, RunRecord } from '../cache/index.js'
 import { isGroupTask, type TaskOutcome } from '../graph/index.js'
 import { VERSION } from '../version.js'
 import type { CiContext, GitContext, HostContext } from './run-context.js'
-import { isCacheHit, taskTelemetryOf, type TaskTelemetry } from './telemetry.js'
+import { taskTelemetryOf, type TaskTelemetry } from './telemetry.js'
+import { isCacheHit } from '../graph/index.js'
 
 export interface RunRecordsInput {
   outcomes: readonly TaskOutcome[]
@@ -27,17 +28,18 @@ export interface RunRecordsInput {
   ok: boolean
   command: string
   requestedTasks: readonly string[]
-  /** Already compacted (`compactCachePolicy`), the form the header row stores. */
+  /** Already formatted (`formatCachePolicy`), the form the header row stores. */
   cachePolicy: string
   concurrency: number
   flow: InvocationRecord['flow']
   forwardArgs?: readonly string[] | undefined
-  tags: Record<string, string>
   git: GitContext
   ci: CiContext
   host: HostContext
   /** Build the telemetry mirror; a run with no sink allocates none. */
   withTelemetry: boolean
+  /** The run's remote failures by key, so far (`Cache.traffic`). */
+  remoteTraffic?: ReadonlyMap<string, { failures: readonly string[] }>
 }
 
 export interface RunRecords {
@@ -46,6 +48,15 @@ export interface RunRecords {
   telemetryTasks: TaskTelemetry[]
 }
 
+/** A key's remote failures as its `runs` row stores them; none when it met none. */
+function remoteErrorOf(
+  traffic: ReadonlyMap<string, { failures: readonly string[] }> | undefined,
+  hash: string | undefined,
+): string | undefined {
+  const failures = hash === undefined ? undefined : traffic?.get(hash)?.failures
+  return failures === undefined || failures.length === 0 ? undefined : failures.join('; ')
+}
+
 /**
  * Every non-group, non-aborted outcome gets a row. An outcome with NO hash
  * (a `skipped` task never probed the cache; a `persistent` one is never
@@ -71,6 +82,7 @@ export function assembleRunRecords(input: RunRecordsInput): RunRecords {
     if (o.status === 'aborted') continue
     if (input.withTelemetry) telemetryTasks.push(taskTelemetryOf(o))
     const blockedBy = o.blockedBy ?? input.tainted?.get(o.node.id)
+    const remoteError = remoteErrorOf(input.remoteTraffic, o.hash)
     runs.push({
       ...(o.hash !== undefined ? { hash: o.hash } : {}),
       project: o.node.projectName,
@@ -113,6 +125,7 @@ export function assembleRunRecords(input: RunRecordsInput): RunRecords {
       ...(o.timedOut === true ? { timedOut: true as const } : {}),
       ...(o.sandboxViolations !== undefined ? { sandboxViolations: o.sandboxViolations } : {}),
       ...(o.notReady !== undefined ? { notReady: o.notReady } : {}),
+      ...(remoteError !== undefined ? { remoteError } : {}),
     })
     if (o.status === 'failed') failedCount++
     if (o.status === 'cache-hit') hitLocalCount++
@@ -151,7 +164,26 @@ export function assembleRunRecords(input: RunRecordsInput): RunRecords {
     os: input.host.os,
     arch: input.host.arch,
     vxVersion: VERSION,
-    tags: JSON.stringify(input.tags),
   }
   return { runs, invocation, telemetryTasks }
 }
+
+/**
+ * The remote failures uploads met after the run's rows went in (the uploads
+ * drain later): each row whose text is now short of its key's is rewritten.
+ */
+export function recordLateRemoteErrors(
+  cache: {
+    readonly traffic: { readonly keys: ReadonlyMap<string, { failures: readonly string[] }> }
+    recordRemoteErrors(runId: string, errors: readonly (readonly [string, string])[]): void
+  },
+  runs: readonly RunRecord[],
+  runId: string,
+): void {
+  const late: [string, string][] = []
+  for (const r of runs) {
+    const now = remoteErrorOf(cache.traffic.keys, r.hash)
+    if (now !== undefined && now !== r.remoteError) late.push([r.hash!, now])
+  }
+  if (late.length > 0) cache.recordRemoteErrors(runId, late)
+}

package/src/orchestrator/run-report.ts

@@ -1,189 +1,91 @@
-// Markdown run report (`vx run --report=markdown` / `--report-file=<path>`).
-// A table — one row per task plus a header line of totals —
-// rendered after a run so CI can put it in a step summary
-// (`vx run ci --report-file="$GITHUB_STEP_SUMMARY"`). Pure: the run's
-// finished outcomes in, a markdown string out. No ANSI, no live region —
-// keep it machine-clean and diffable.
-//
-// Machine-clean is a property of this STRING, not of stdout: the status
-// logger shares that stream, so redirecting it captures the whole run log
-// too. `--report-file` is the sink that carries the property end to end.
+// The report every plugin's teardown reads, and the run's own bytes kept
+// in a remote that takes its writes (owner 2026-10-15): a failed task's
+// log and the files its outputs left, an uncached task's log.
 
-import { failedLabel, skippedLabel, type OutcomeView } from './events.js'
-import { neverStarted } from './summary.js'
+import {
+  packRunOutputs,
+  resolveOutputs,
+  resolveWorkspaceOutputs,
+  runEntryName,
+} from '../cache/index.js'
+import type { CiContext } from './run-context.js'
+import { isGroupTask, type TaskNode, type TaskOutcome } from '../graph/index.js'
+import { relPosix } from '../util/index.js'
+import { decodeOutputLog } from './output-log.js'
+import type { RunReport, RunReportTask } from './plugin.js'
 
-/**
- * One finished run, reduced to what a report needs. Used to live in
- * `protocol.ts` as the return type of the whole-run `backend` seam; that
- * seam is gone (a run always executes in-process — see
- * `docs/modules/executor.md`), so the shape lives with its only consumer.
- */
-export interface RunResult {
-  ok: boolean
-  outcomes: OutcomeView[]
+export interface RunReportArgs {
+  readonly runId: string
+  readonly commit: string | null
+  readonly ciRun: CiContext
+  readonly ok: boolean
+  readonly workspaceRoot: string
+  readonly nestedDirsByProject: ReadonlyMap<string, string[]>
+  /** Stores one run entry; undefined when no remote takes the run's writes. */
+  readonly keep: ((name: string, bytes: Uint8Array) => Promise<boolean>) | undefined
 }
-import { tallyViews, type Tally } from './tally.js'
 
-/**
- * The report header's numbers. The outcome BUCKETS come from the shared
- * `tallyViews` — the same rules the terminal summary and `--summarize` use,
- * so three surfaces describing one run cannot disagree. Only the two
- * durations are report-specific.
- *
- * This used to be a private copy of the partition, and it drifted: it had no
- * group filter (it could not have one — `OutcomeView` had no `isGroup`), so
- * every organizational node was counted as a successful task and rendered as
- * a row claiming `success | miss | 0ms`.
- */
-interface ReportTally extends Tally {
-  /** Wall-clock ms summed over tasks that ran a command, an aborted one's time so far included. */
-  executedMs: number
-  /** The aborted tasks the stop reached before they ran, as the terminal splits them. */
-  notRun: number
-  /**
-   * Ms of work the cache SKIPPED — the sum of the hits' STORED exec times.
-   * Deliberately not `durationMs`, which for a hit is the restore this run
-   * paid: summing that reported a 2.01 s task as "6ms saved".
-   */
-  savedMs: number
+/** The files `node`'s declared outputs match on disk now, workspace-relative and sorted. */
+async function outputsLeft(
+  node: TaskNode,
+  workspaceRoot: string,
+  nestedProjectDirs: string[],
+): Promise<string[]> {
+  const declared = node.config.cache?.outputs
+  if (declared === undefined) return []
+  const [own, ws] = await Promise.all([
+    resolveOutputs({ projectDir: node.projectDir, outputs: declared.files, nestedProjectDirs }),
+    resolveWorkspaceOutputs({ workspaceRoot, outputs: declared.workspaceFiles ?? [] }),
+  ])
+  return [...new Set([...own, ...ws].map((p) => relPosix(workspaceRoot, p)))].sort()
 }
 
-function tally(outcomes: readonly OutcomeView[]): ReportTally {
-  const t: ReportTally = { ...tallyViews(outcomes), executedMs: 0, savedMs: 0, notRun: 0 }
-  for (const o of outcomes) {
-    if (o.isGroup === true) continue
-    switch (o.status) {
-      case 'success':
-      case 'failed':
-        t.executedMs += o.durationMs
-        break
-      case 'aborted':
-        if (neverStarted(o)) t.notRun++
-        else t.executedMs += o.durationMs
-        break
-      case 'cache-hit':
-      case 'cache-hit-remote':
-        // An outcome that doesn't know what it skipped contributes nothing:
-        // better to omit the claim than to substantiate it with the restore
-        // cost. (Only reachable across a version skew — every hit this
-        // binary produces carries the stored duration.)
-        t.savedMs += o.storedDurationMs ?? 0
-        break
+export async function keepRunBytes(
+  final: readonly TaskOutcome[],
+  args: RunReportArgs,
+): Promise<RunReport> {
+  const { keep, runId, workspaceRoot } = args
+  const tasks: RunReportTask[] = []
+  const work: (() => Promise<void>)[] = []
+  for (const o of final) {
+    if (isGroupTask(o.node)) continue
+    const task: { -readonly [K in keyof RunReportTask]: RunReportTask[K] } = {
+      id: o.node.id,
+      status: o.status,
+      durationMs: Math.round(o.durationMs),
+      ...(o.hash !== undefined ? { key: o.hash } : {}),
+    }
+    tasks.push(task)
+    if (keep === undefined) continue
+    const runLog = o.runLog
+    if (runLog !== undefined) {
+      work.push(async () => {
+        const text = decodeOutputLog(runLog)
+          .map((c) => c.text)
+          .join('')
+        if (text.length === 0) return
+        const name = runEntryName(runId, o.node.id, 'log')
+        if (await keep(name, new TextEncoder().encode(text))) task.log = name
+      })
+    }
+    if (o.status === 'failed' && o.where === undefined) {
+      work.push(async () => {
+        const bytes = await packRunOutputs(
+          workspaceRoot,
+          await outputsLeft(
+            o.node,
+            workspaceRoot,
+            args.nestedDirsByProject.get(o.node.projectName) ?? [],
+          ),
+        )
+        if (bytes === null) return
+        const name = runEntryName(runId, o.node.id, 'outputs')
+        if (await keep(name, bytes)) task.outputs = name
+      })
     }
   }
-  return t
-}
-
-function statusWord(o: OutcomeView): string {
-  switch (o.status) {
-    case 'success':
-      return 'success'
-    case 'cache-hit':
-    case 'cache-hit-remote':
-      return 'success'
-    case 'failed':
-      return failedLabel(o.exitCode, o.timedOut, o.sandboxViolations, o.notReady, o.inputMoved)
-    case 'skipped':
-      return skippedLabel(o.blockedBy)
-    case 'aborted':
-      return neverStarted(o) ? 'not run' : 'aborted'
-    default:
-      return o.status
-  }
-}
-
-function cacheWord(o: OutcomeView): string {
-  switch (o.status) {
-    case 'cache-hit':
-      return o.restored === false ? 'up-to-date' : 'local'
-    case 'cache-hit-remote':
-      return o.restored === false ? 'up-to-date' : 'remote'
-    case 'success':
-    case 'failed':
-      return o.noCache === true ? 'no-cache' : 'miss'
-    default:
-      return '—'
-  }
-}
-
-function fmtDuration(ms: number): string {
-  if (ms < 1000) return `${Math.round(ms)}ms`
-  return `${(ms / 1000).toFixed(2)}s`
-}
-
-/**
- * Make a value safe inside a GFM table cell. Task names are arbitrary TS
- * object keys and the loader accepts `|` and newlines, either of which
- * silently breaks the table on the consumer (`>> $GITHUB_STEP_SUMMARY`):
- * a bare pipe adds a column, a newline splits the row.
- *
- * Exported because this file is not the only markdown table describing a run:
- * the cloud plugin's GitHub job summary renders the same data from the same
- * unvalidated names and shipped WITHOUT this escape, so a `|` in a task name
- * or an output path shifted its columns. One definition, so the two
- * cannot disagree about what a cell may contain.
- *
- * A pipe splits the row unless an ODD run of backslashes precedes it (GFM
- * reads `\\` as one escaped backslash), so a pipe after an odd run is already
- * escaped and one more backslash would free it: `a\|b` became `a\\|b`, two
- * cells. A lone `\r` ends a line too.
- */
-export function escapeMarkdownCell(value: string): string {
-  return value
-    .replace(/(\\*)\|/g, (pipe, run: string) => (run.length % 2 === 0 ? `${run}\\|` : pipe))
-    .replace(/\r\n|\r|\n/g, ' ')
-}
-
-const cell = escapeMarkdownCell
-
-/**
- * Render the finished run as a markdown report. A header line of totals
- * followed by a per-task table. Self-contained — everything is derived
- * from the result's outcomes.
- */
-export function formatRunReportMarkdown(result: RunResult): string {
-  const t = tally(result.outcomes)
-  const cached = t.cachedLocal + t.cachedRemote
-  // The terminal summary's words and counts: every task in the run counts
-  // (M, 2026-10-14), and a task the stop reached before it ran was never
-  // aborted.
-  const aborted = t.aborted - t.notRun
-  const count = t.total + aborted + t.notRun
-  const parts = [
-    `**${count} task${count === 1 ? '' : 's'}**`,
-    `${t.successful} success`,
-    `${t.failed} failed`,
-    // What the hits did to the disk, as the run's summary splits them.
-    cached > 0
-      ? `${cached} cached (${t.upToDate} up-to-date, ${t.restoredLocal + t.restoredRemote} restored)`
-      : '0 cached',
-  ]
-  if (t.skipped > 0) parts.push(`${t.skipped} skipped`)
-  if (aborted > 0) parts.push(`${aborted} aborted`)
-  if (t.notRun > 0) parts.push(`${t.notRun} not run`)
-  parts.push(`${fmtDuration(t.executedMs)} total`)
-  if (t.savedMs > 0) parts.push(`${fmtDuration(t.savedMs)} saved`)
-
-  // Only a stop (a signal, `RunOptions.signal`) makes an `aborted` outcome,
-  // so one is how an interrupted run tells itself from a failed one.
-  const heading = t.aborted > 0 ? 'interrupted' : result.ok ? 'passed' : 'failed'
-  const lines: string[] = []
-  lines.push(`## vx run — ${heading}`)
-  lines.push('')
-  lines.push(parts.join(' · '))
-  lines.push('')
-  lines.push('| Task | Status | Cache | Duration |')
-  lines.push('| --- | --- | --- | --- |')
-  for (const o of result.outcomes) {
-    // A group node did no work: no command, no cache decision, 0ms. Rendering
-    // one as `success | miss | 0ms` invents a task the user never wrote a
-    // command for. Same exclusion the header's tally makes.
-    if (o.isGroup === true) continue
-    lines.push(
-      `| ${cell(o.taskId)} | ${cell(statusWord(o))} | ${cell(cacheWord(o))} | ${cell(
-        neverStarted(o) ? '—' : fmtDuration(o.durationMs),
-      )} |`,
-    )
+  for (let i = 0; i < work.length; i += 8) {
+    await Promise.all(work.slice(i, i + 8).map((w) => w()))
   }
-  return lines.join('\n') + '\n'
+  return { runId, commit: args.commit, ciRun: args.ciRun, ok: args.ok, tasks }
 }

package/src/orchestrator/run-settings.ts

@@ -5,7 +5,6 @@
 
 import type { WorkspaceConfig } from '../config.js'
 import { resolveSetting } from '../settings.js'
-import { setSettleBounds, type SettleBound } from '../util/index.js'
 import type { RunOptions } from './options.js'
 
 const FROM_OPTIONS = {
@@ -14,15 +13,12 @@ const FROM_OPTIONS = {
   timeout: 'run.timeout',
   retries: 'run.retries',
   continueMode: 'run.continue',
-  readyNotice: 'run.readyNotice',
   download: 'cache.download',
 } as const satisfies Partial<Record<keyof RunOptions, string>>
 
 /**
  * `options` with each run-level setting it leaves unset filled from the
- * environment, then the workspace. Arms the settle bounds (`util/settle`,
- * below settings) with the flag's or workspace's `run.killGrace` and
- * `run.teardownTimeout`: the variable sits between them, read per call.
+ * environment, then the workspace.
  */
 export function withRunSettings(
   options: RunOptions,
@@ -33,16 +29,5 @@ export function withRunSettings(
     const { value } = resolveSetting(path, workspace, options[field as keyof RunOptions])
     if (value !== undefined) out[field] = value
   }
-  const bound = (flag: number | undefined, path: string): SettleBound | undefined => {
-    if (flag !== undefined) return { ms: flag, from: 'flag' }
-    const { value } = resolveSetting(path, workspace, undefined, {})
-    return value === undefined ? undefined : { ms: value as number, from: 'workspace' }
-  }
-  const killGrace = bound(options.killGrace, 'run.killGrace')
-  const teardown = bound(options.teardownTimeout, 'run.teardownTimeout')
-  setSettleBounds({
-    ...(killGrace !== undefined ? { killGrace } : {}),
-    ...(teardown !== undefined ? { teardown } : {}),
-  })
   return out as unknown as RunOptions
 }

package/src/orchestrator/run-stop.ts

@@ -0,0 +1,153 @@
+// One stop for the run: an embedder's `RunOptions.signal` and a process
+// signal both abort it. The scheduler stops dispatching (it reads the
+// signal), the children are torn down, and run() returns through its own
+// end-of-run path. Owns the run-scoped registries of live subprocesses:
+//   - `liveChildren`: in-flight children. The runner adds/removes each
+//     child around its spawn (persistent children stay until they exit).
+//   - `persistentRegistry`: ready persistent tasks (dev servers,
+//     watchers). executeTask spawns them but does NOT await their exit;
+//     ownership moves here so the orchestrator can SIGTERM them once the
+//     rest of the graph finishes.
+// A SIGINT/SIGTERM mid-run forwards that signal to everything in both
+// (`signals.ts`); `detach` removes the handlers so repeated run() calls
+// (test suites) never stack listeners.
+
+import { killGraceMs, teardownTimeoutMs } from '../util/index.js'
+import { type Logger, warnOf } from '../report/index.js'
+import { hasEnded } from './persistent.js'
+import {
+  forwardedSignal,
+  forwardSignals,
+  SIGNAL_SHUTDOWN_GRACE_MS,
+  type StopSignal,
+  terminateChildren,
+} from './signals.js'
+
+type Child = ReturnType<typeof Bun.spawn>
+
+export interface RunStop {
+  readonly signal: AbortSignal
+  readonly liveChildren: Set<Child>
+  readonly persistentRegistry: Map<string, Child>
+  /**
+   * The process signal that stopped the run. A run a process signal
+   * stopped records no history: until item 849 the handler exited before
+   * any outcome landed, and a Ctrl-C'd run then read in `vx last` as
+   * FAILED with 0 tasks (item 854). An embedder's abort is its own call
+   * and keeps its record.
+   */
+  readonly stoppedBy: StopSignal | undefined
+  /**
+   * The stop's teardown of every child, awaited before run() leaves: a
+   * task's shell can die of the signal at once while a child it
+   * backgrounded ignores it and dies only to the SIGKILL after the grace,
+   * and a run that returned first left that child running past vx's exit
+   * (turborepo#14043's class).
+   */
+  readonly aborting: Promise<void> | undefined
+  /**
+   * The servers that had ended when the stop landed: those crashed. One
+   * that ends after it is the stop's own kill, and was named a crash
+   * (item 1061).
+   */
+  readonly endedBeforeStop: ReadonlySet<Child> | undefined
+  /** The graph finished: a server ending from here on is said by the end of the run. */
+  graphDone(): void
+  /** Removes the signal handlers and abort listeners. */
+  detach(): void
+  /** run() has left its `finally`: a process signal waits for this before it exits. */
+  left(): void
+}
+
+export function runStop(args: {
+  log: Logger
+  cache: { close(): void }
+  /** `RunOptions.signal`. */
+  external: AbortSignal | undefined
+  /** `RunOptions.handleSignals`. */
+  handleSignals: boolean
+  handsTerminal: boolean
+  /** How many plugins have a teardown: each gets its own bound before a signal exits. */
+  teardowns: number
+  /** Called first when the run is stopped. */
+  onStop: () => void
+}): RunStop {
+  const { log, external } = args
+  const liveChildren = new Set<Child>()
+  // A server that ends on its own while the graph still runs is said at
+  // once: its dependants were failing against it, and the end of the run
+  // was the first word of it (C-69). Not once the graph is done (the end
+  // of the run and the keep-alive wait say it) nor under a stop.
+  let graphDone = false
+  const persistentRegistry = new (class extends Map<string, Child> {
+    override set(id: string, child: Child): this {
+      super.set(id, child)
+      void child.exited.then((code) => {
+        if (code !== 0 && !graphDone && !controller.signal.aborted)
+          warnOf(log)(`vx: ${id} exited with code ${code} while the run went on`)
+      })
+      return this
+    }
+  })()
+  const controller = new AbortController()
+  const forwardAbort = (): void => controller.abort(external?.reason)
+  if (external?.aborted === true) forwardAbort()
+  else external?.addEventListener('abort', forwardAbort, { once: true })
+  let stoppedBy: StopSignal | undefined
+  let left = (): void => {}
+  const runLeft = new Promise<void>((resolve) => {
+    left = resolve
+  })
+  const signals = forwardSignals({
+    enabled: args.handleSignals,
+    log,
+    cache: args.cache,
+    liveChildren,
+    persistentRegistry,
+    stop: (signal) => {
+      stoppedBy = signal
+      controller.abort(signal)
+    },
+    done: runLeft,
+    handsTerminal: args.handsTerminal,
+    // The grace, then one flush and each teardown at their own bound, and
+    // slack for the summary and the cache close.
+    boundMs:
+      killGraceMs(SIGNAL_SHUTDOWN_GRACE_MS) + teardownTimeoutMs() * (1 + args.teardowns) + 2_000,
+  })
+  let aborting: Promise<void> | undefined
+  let endedBeforeStop: ReadonlySet<Child> | undefined
+  const onAbort = (): void => {
+    args.onStop()
+    endedBeforeStop = new Set([...persistentRegistry.values()].filter(hasEnded))
+    aborting = terminateChildren(
+      () => [...liveChildren, ...persistentRegistry.values()],
+      forwardedSignal(controller.signal.reason),
+    )
+  }
+  if (controller.signal.aborted) onAbort()
+  else controller.signal.addEventListener('abort', onAbort, { once: true })
+  return {
+    signal: controller.signal,
+    liveChildren,
+    persistentRegistry,
+    get stoppedBy() {
+      return stoppedBy
+    },
+    get aborting() {
+      return aborting
+    },
+    get endedBeforeStop() {
+      return endedBeforeStop
+    },
+    graphDone: () => {
+      graphDone = true
+    },
+    detach: () => {
+      signals.remove()
+      external?.removeEventListener('abort', forwardAbort)
+      controller.signal.removeEventListener('abort', onAbort)
+    },
+    left: () => left(),
+  }
+}