@vzn/vx 0.0.518 → 0.0.520

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vzn/vx",
-  "version": "0.0.518",
+  "version": "0.0.520",
   "description": "The fastest task runner and build cache for JavaScript monorepos.",
   "type": "module",
   "exports": {
@@ -21,10 +21,10 @@
     "node": ">=18"
   },
   "optionalDependencies": {
-    "@vzn/vx-linux-x64": "0.0.518",
-    "@vzn/vx-linux-arm64": "0.0.518",
-    "@vzn/vx-darwin-x64": "0.0.518",
-    "@vzn/vx-darwin-arm64": "0.0.518"
+    "@vzn/vx-linux-x64": "0.0.520",
+    "@vzn/vx-linux-arm64": "0.0.520",
+    "@vzn/vx-darwin-x64": "0.0.520",
+    "@vzn/vx-darwin-arm64": "0.0.520"
   },
   "dependencies": {
     "@anthropic-ai/sandbox-runtime": "^0.0.76"

package/src/exec/executor.ts

@@ -152,6 +152,13 @@ export interface ExecuteRequest {
   readonly signal?: AbortSignal
   /** See `RunOptions.liveChildren`: the run's SIGINT/SIGTERM registry. */
   readonly liveChildren?: Set<ReturnType<typeof Bun.spawn>>
+  /**
+   * An executor that runs the command as a process on THIS machine calls
+   * this with its pid once spawned, so a telemetry plugin can sample the
+   * process tree's CPU and memory while it runs. Absent when nobody samples;
+   * an executor that runs elsewhere never calls it.
+   */
+  readonly onSpawn?: (pid: number) => void
   readonly sandbox?: ExecuteSandbox
   /**
    * The task holds vx's terminal (`exec.interactive` on a TTY): its stdin,

package/src/exec/index.ts

@@ -45,6 +45,7 @@ export {
 export { bindableWrites, punchWalls } from './sandbox-binds.js'
 export { isLocalExecutor, localExecutor } from './local-executor.js'
 export { holdGroups, killTree, untilGroupsGone } from './kill-tree.js'
+export { sampleTrees } from './proc-sample.js'
 export {
   assertExecuteResult,
   executorFallback,

package/src/exec/local-executor.ts

@@ -42,6 +42,7 @@ export function localExecutor(): TaskExecutor {
         onStderr: req.onStderr,
         capture: req.capture,
         ...(req.liveChildren !== undefined ? { liveChildren: req.liveChildren } : {}),
+        ...(req.onSpawn !== undefined ? { onSpawn: req.onSpawn } : {}),
         ...(req.timeoutMs !== undefined ? { timeoutMs: req.timeoutMs } : {}),
       }
       if (req.sandbox === undefined) {

package/src/exec/proc-sample.ts

@@ -0,0 +1,151 @@
+// What a running task's process tree uses right now: CPU time and resident
+// memory summed over a root pid and every descendant. `resourceUsage()`
+// answers only at exit, so a task's curve needs a look while it runs.
+//
+// A tree, not a process group: a sandboxed task runs under bwrap in a
+// session of its own, so its processes carry another group, but each still
+// has its parent. A process that left the tree (re-parented to init) is not
+// counted; nor is the CPU of a descendant that already exited.
+
+import { readdir, readFile } from 'node:fs/promises'
+import { procfsIsOwn } from '../util/index.js'
+
+export interface TreeUsage {
+  /** CPU time of the live tree, user + system, in ms. */
+  cpuMs: number
+  /** Resident memory of the live tree, in bytes. */
+  rssBytes: number
+}
+
+interface Proc {
+  ppid: number
+  cpuMs: number
+  /** Resident KiB when the table carries it; Linux reads it per tree member. */
+  rssKiB?: number
+}
+
+/**
+ * Linux's `/proc/<pid>/stat` counts CPU in USER_HZ ticks, fixed at 100 by
+ * the ABI on every architecture; `tests/proc-sample.test.ts` pins it by
+ * burning a known span and reading it back.
+ */
+const TICK_MS = 10
+
+/** Each root's tree usage; a root no longer alive has no entry. */
+export async function sampleTrees(roots: readonly number[]): Promise<Map<number, TreeUsage>> {
+  const out = new Map<number, TreeUsage>()
+  if (roots.length === 0) return out
+  const table = process.platform === 'linux' ? await linuxTable() : await psTable()
+  if (table === undefined) return out
+  const children = new Map<number, number[]>()
+  for (const [pid, p] of table) {
+    const list = children.get(p.ppid)
+    if (list === undefined) children.set(p.ppid, [pid])
+    else list.push(pid)
+  }
+  for (const root of roots) {
+    if (!table.has(root)) continue
+    const tree: number[] = []
+    const stack = [root]
+    while (stack.length > 0) {
+      const pid = stack.pop()!
+      tree.push(pid)
+      for (const c of children.get(pid) ?? []) stack.push(c)
+    }
+    let cpuMs = 0
+    let rssKiB = 0
+    const rss = await Promise.all(tree.map((pid) => table.get(pid)!.rssKiB ?? linuxRssKiB(pid)))
+    for (const [i, pid] of tree.entries()) {
+      cpuMs += table.get(pid)!.cpuMs
+      rssKiB += rss[i]!
+    }
+    out.set(root, { cpuMs, rssBytes: rssKiB * 1024 })
+  }
+  return out
+}
+
+async function linuxTable(): Promise<Map<number, Proc> | undefined> {
+  // Another namespace's /proc is a table of strangers: no answer beats a wrong one.
+  if (!procfsIsOwn()) return undefined
+  let entries: string[]
+  try {
+    entries = await readdir('/proc')
+  } catch {
+    return undefined
+  }
+  const table = new Map<number, Proc>()
+  await Promise.all(
+    entries.map(async (entry) => {
+      const first = entry.charCodeAt(0)
+      if (first < 48 || first > 57) return
+      let stat: string
+      try {
+        stat = await readFile(`/proc/${entry}/stat`, 'utf8')
+      } catch {
+        return // exited between the listing and the read
+      }
+      // `<pid> (<comm>) <state> <ppid> …` — comm may hold spaces and parens,
+      // so the fields start after the LAST ')': state is [0], ppid [1],
+      // utime [11], stime [12].
+      const f = stat.slice(stat.lastIndexOf(')') + 2).split(' ')
+      if (f[0] === 'Z' || f[0] === 'X') return
+      table.set(Number(entry), {
+        ppid: Number(f[1]),
+        cpuMs: (Number(f[11]) + Number(f[12])) * TICK_MS,
+      })
+    }),
+  )
+  return table
+}
+
+/** `VmRSS` carries its unit (kB); 0 for a process gone or a kernel thread. */
+async function linuxRssKiB(pid: number): Promise<number> {
+  try {
+    const status = await readFile(`/proc/${pid}/status`, 'utf8')
+    const m = /^VmRSS:\s+(\d+) kB$/m.exec(status)
+    return m === null ? 0 : Number(m[1])
+  } catch {
+    return 0
+  }
+}
+
+/** macOS (and any non-Linux): one `ps` for the whole table. */
+async function psTable(): Promise<Map<number, Proc> | undefined> {
+  let text: string
+  try {
+    const ps = Bun.spawn(['ps', '-A', '-o', 'pid=,ppid=,rss=,time='], {
+      stdin: 'ignore',
+      stdout: 'pipe',
+      stderr: 'ignore',
+    })
+    text = await new Response(ps.stdout).text()
+    if ((await ps.exited) !== 0) return undefined
+  } catch {
+    return undefined
+  }
+  const table = new Map<number, Proc>()
+  for (const line of text.split('\n')) {
+    const f = line.trim().split(/\s+/)
+    if (f.length < 4) continue
+    const cpuMs = psTimeMs(f[3]!)
+    if (cpuMs === undefined) continue
+    table.set(Number(f[0]), { ppid: Number(f[1]), rssKiB: Number(f[2]), cpuMs })
+  }
+  return table
+}
+
+/** `ps`'s `time`: `[[dd-]hh:]mm:ss[.cc]`, macOS printing hundredths. */
+export function psTimeMs(s: string): number | undefined {
+  const dash = s.indexOf('-')
+  const days = dash === -1 ? 0 : Number(s.slice(0, dash))
+  const parts = s
+    .slice(dash + 1)
+    .split(':')
+    .map(Number)
+  if (parts.length < 2 || parts.some((n) => !Number.isFinite(n)) || !Number.isFinite(days)) {
+    return undefined
+  }
+  let seconds = 0
+  for (const n of parts) seconds = seconds * 60 + n
+  return Math.round((days * 86_400 + seconds) * 1000)
+}

package/src/exec/runner.ts

@@ -87,6 +87,8 @@ export interface RunOptions {
    * handler can SIGTERM everything still alive mid-run.
    */
   liveChildren?: Set<ReturnType<typeof Bun.spawn>>
+  /** Told the child's pid once it is spawned (telemetry samples its tree). */
+  onSpawn?: (pid: number) => void
   /**
    * Hand the child vx's own stdin, stdout and stderr (`exec.interactive` on
    * a TTY). Nothing passes through vx: the callbacks hear nothing and the
@@ -632,6 +634,7 @@ export function runPersistent(opts: PersistentOptions): PersistentSpawn {
   void consumeChunks(child.stderr, true)
 
   opts.liveChildren?.add(child)
+  opts.onSpawn?.(child.pid)
 
   // Readiness deadline. Reject FIRST so the failure reads as a
   // timeout, then SIGTERM — the exit handler's later reject is a
@@ -738,6 +741,7 @@ export async function runCommand(opts: RunOptions): Promise<RunResult> {
   }
 
   opts.liveChildren?.add(proc)
+  opts.onSpawn?.(proc.pid)
   const timeout = armTimeout(proc, opts.timeoutMs)
   const ac = new AbortController()
   const streams = Promise.all([

package/src/exec/sandbox-runtime.ts

@@ -808,6 +808,8 @@ export interface SandboxedRunArgs {
   onStderr?: (chunk: string) => void
   /** See `RunOptions.liveChildren` — same contract for sandboxed spawns. */
   liveChildren?: Set<ReturnType<typeof Bun.spawn>>
+  /** See `RunOptions.onSpawn`. */
+  onSpawn?: (pid: number) => void
   /** `ExecuteRequest.signal`: once aborted, the tracer retry spawns nothing. */
   signal?: AbortSignal
   /** See `RunOptions.timeoutMs` — SIGTERM the child after this many ms. */
@@ -1836,6 +1838,7 @@ async function runSandboxedOnce(
   }
 
   args.liveChildren?.add(proc)
+  args.onSpawn?.(proc.pid)
   const timeout = armTimeout(proc, args.timeoutMs)
   const ac = new AbortController()
   // The unfinished last line of stderr, and whether a line was strace's.

package/src/graph/scheduler.ts

@@ -139,6 +139,8 @@ export interface TaskOutcome {
    * flaky detection reads it off the outcome stream.
    */
   attempts?: number
+  /** The attempts that failed and were run again, in order: when each ended (epoch ms) and how. */
+  failedAttempts?: readonly { endedAt: number; exitCode: number; timedOut?: true }[]
   /**
    * Set when this run proved the task flaky: its key holds both outcomes on
    * record (this run's included) or it needed a retry. Its row carries it.

package/src/orchestrator/execute-task.ts

@@ -145,6 +145,8 @@ export interface ExecuteArgs {
    * SIGTERMs whatever is in here.
    */
   liveChildren?: Set<ReturnType<typeof Bun.spawn>>
+  /** Sample a spawned task's process tree (`TelemetrySource.track`); absent when nobody samples. */
+  track?: (taskId: string, pid: number) => () => void
   /**
    * Run-level retry default (`--retry <n>` / `RunOptions.retries`).
    * Explicit `exec.retries` wins, including an explicit 0. Threaded as
@@ -376,6 +378,9 @@ async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutcome> {
     onStdout: serverOut ? (chunk) => serverOut.push(chunk) : (chunk) => log.taskStdout(node, chunk),
     onStderr: serverErr ? (chunk) => serverErr.push(chunk) : (chunk) => log.taskStderr(node, chunk),
     ...(args.liveChildren !== undefined ? { liveChildren: args.liveChildren } : {}),
+    ...(args.track !== undefined
+      ? { onSpawn: (pid: number) => void args.track!(node.id, pid) }
+      : {}),
     ...(signalChannel ? { signalChannel } : {}),
     ...(args.terminal === true ? { terminal: true } : {}),
   }
@@ -953,6 +958,8 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
         unlistenStop?.()
         unlistenStop = undefined
         flushMasked()
+        untrack?.()
+        untrack = undefined
       })
     endExec()
     if (secrets !== null)
@@ -1076,6 +1083,9 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
   // and what the cache keeps of it (L-11); null when there are none.
   const secrets = secretMask([process.env, env, step.env?.define], step.env?.secret)
   let flushMasked = (): void => {}
+  const failedAttempts: { endedAt: number; exitCode: number; timedOut?: true }[] = []
+  // The sampling of the attempt's process tree, stopped once it settles.
+  let untrack: (() => void) | undefined
   // The entry's command is shown by `vx why` and sent with the entry to a
   // remote cache: a value a config interpolated stays out of both.
   const storedCommand = secrets?.mask(step.command) ?? step.command
@@ -1130,6 +1140,11 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
     }
 
     if (effectiveExitCode === 0 || attempt >= maxAttempts) break
+    failedAttempts.push({
+      endedAt: Date.now(),
+      exitCode: effectiveExitCode,
+      ...(result.timedOut === true ? { timedOut: true as const } : {}),
+    })
     log.taskStderr(
       node,
       `vx: retrying ${node.id} (attempt ${attempt + 1}/${maxAttempts}) after ${result.timedOut === true ? 'a timeout' : `exit ${effectiveExitCode}`}\n`,
@@ -1155,6 +1170,14 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
       onStdout: out ? (chunk) => out.push(chunk) : (chunk) => log.taskStdout(node, chunk),
       onStderr: err ? (chunk) => err.push(chunk) : (chunk) => log.taskStderr(node, chunk),
       ...(args.liveChildren !== undefined ? { liveChildren: args.liveChildren } : {}),
+      ...(args.track !== undefined
+        ? {
+            onSpawn: (pid: number) => {
+              untrack?.()
+              untrack = args.track!(node.id, pid)
+            },
+          }
+        : {}),
       signal: requestSignal(),
       ...(effectiveTimeout !== undefined ? { timeoutMs: effectiveTimeout } : {}),
       ...(inputs !== undefined ? { inputs } : {}),
@@ -1337,6 +1360,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
     durationMs: spentMs,
     hash,
     ...(attempt > 1 ? { attempts: attempt } : {}),
+    ...(failedAttempts.length > 0 ? { failedAttempts } : {}),
     ...(unkeyed ? { unkeyed: true as const } : {}),
     ...(result.timedOut === true && effectiveExitCode !== 0 ? { timedOut: true as const } : {}),
     ...(result.cpuMs !== undefined ? { cpuMs: result.cpuMs } : {}),

package/src/orchestrator/prepare.ts

@@ -9,7 +9,7 @@
 
 import path from 'node:path'
 import type { WorkspaceConfig } from '../config.js'
-import { mark, nearest, UserError } from '../util/index.js'
+import { beginRun, mark, nearest, UserError } from '../util/index.js'
 import {
   Cache,
   noteSchemaReset,
@@ -182,6 +182,7 @@ export interface PreparedRun {
  * empty plan).
  */
 export async function prepareRun(options: RunOptions, log: Logger): Promise<PreparedRun> {
+  beginRun()
   mark('startup')
   // The root manifest is read once for the root, the globs and the
   // fingerprint. A watch cycle is a new run and reads it afresh.

package/src/orchestrator/run.ts

@@ -26,6 +26,7 @@ import {
 } from '../graph/index.js'
 import {
   mark,
+  stageTimes,
   killGraceMs,
   MAX_TIMEOUT_MS,
   parseDuration,
@@ -755,6 +756,7 @@ async function runOnBus(
         { workspaceRoot, cacheDir, warn: (m: string) => log.status(m) },
         runContextRecord,
         options.telemetrySinks,
+        nodes,
       )
     }
 
@@ -938,6 +940,7 @@ async function runOnBus(
         runStartHrTimeNs,
         persistentRegistry,
         liveChildren,
+        ...(telemetry?.track !== undefined ? { track: telemetry.track } : {}),
         gitFilesCache,
         hashCache,
         ...(probe !== undefined ? { preProbed: probe } : {}),
@@ -1201,6 +1204,7 @@ async function runOnBus(
         totalDurationMs: Math.round(totalMs),
         exitOk: ok,
         abortedCount: list.filter((o) => o.status === 'aborted' && !isGroupTask(o.node)).length,
+        stages: stageTimes(),
       })
       telemetry.emitSummary(summary)
       await telemetry.flush()

package/src/orchestrator/telemetry-host.ts

@@ -12,6 +12,7 @@
 // See docs/design/observability-architecture-2026-06.md §1, §3.
 
 import type { EventBus } from './events.js'
+import type { TaskNode } from '../graph/index.js'
 import type {
   RunContextRecord,
   RunSummaryRecord,
@@ -30,6 +31,8 @@ export interface TelemetryHandle {
   flush(): Promise<void>
   /** Remove the bus subscription. Idempotent. */
   dispose(): void
+  /** Sample a running task's process tree; see `TelemetrySource.track`. */
+  readonly track?: (taskId: string, pid: number) => () => void
 }
 
 /** Reject anything that is not sink-shaped, naming what arrived. */
@@ -68,6 +71,7 @@ export async function subscribeTelemetry(
   ctx: TelemetryContext,
   run: RunContextRecord,
   extraSinks?: readonly TelemetrySink[],
+  nodes?: ReadonlyMap<string, TaskNode>,
 ): Promise<TelemetryHandle | undefined> {
   const sinks: TelemetrySink[] = extraSinks === undefined ? [] : [...extraSinks]
   const owners = new Map<TelemetrySink, string>()
@@ -109,7 +113,13 @@ export async function subscribeTelemetry(
 
   if (sinks.length === 0) return undefined
 
-  const source = createTelemetrySource({ sinks, run, warn: (m) => ctx.warn(m), owners })
+  const source = createTelemetrySource({
+    sinks,
+    run,
+    warn: (m) => ctx.warn(m),
+    owners,
+    ...(nodes !== undefined ? { nodes } : {}),
+  })
   // The bus's unsubscribe is idempotent, so it is the handle's dispose as
   // is (a once-flag here survived item 654).
   const dispose = bus.subscribe(source.subscriber)
@@ -117,5 +127,6 @@ export async function subscribeTelemetry(
     emitSummary: (summary) => source.emitSummary(summary),
     flush: () => source.flush(),
     dispose,
+    ...(source.track !== undefined ? { track: source.track } : {}),
   }
 }

package/src/orchestrator/telemetry.ts

@@ -16,7 +16,8 @@
 // what or how tasks run. Contrast `cache`/`executor`, which return objects
 // core calls INTO; those are the behavior capabilities, kept separate.
 
-import type { TaskOutcome, TaskStatus } from '../graph/index.js'
+import { sampleTrees } from '../exec/index.js'
+import { isGroupTask, type TaskNode, type TaskOutcome, type TaskStatus } from '../graph/index.js'
 import { maskedCommand, settleWithin, teardownTimeoutMs } from '../util/index.js'
 import type { RunEvent, RunEventSubscriber } from './events.js'
 
@@ -145,6 +146,15 @@ export interface RunContextRecord {
   tags: Readonly<Record<string, string>>
 }
 
+/** One attempt of a retried task that failed and was run again. */
+export interface FailedAttempt {
+  /** When the attempt ended, epoch ms. */
+  endedAt: number
+  exitCode: number
+  /** vx's own `timeout` ended it. */
+  timedOut?: true
+}
+
 /** Denormalized per-task analytics — shared by the streaming `task.end`
  *  record and the per-run summary's `tasks[]`. */
 export interface TaskTelemetry {
@@ -180,6 +190,8 @@ export interface TaskTelemetry {
   sandboxViolations?: number
   /** On a failed persistent task: why it never became ready. `exitCode` is the child's own when it exited. Additive. */
   notReady?: 'timeout' | 'exited' | 'spawn'
+  /** On a task that retried: each attempt that failed before the last, in order. Additive. */
+  failedAttempts?: readonly FailedAttempt[]
   /**
    * On a cache hit: whether outputs were written this run (`true`) or the
    * disk already matched the entry and nothing was restored (`false`, an
@@ -216,6 +228,12 @@ export type TelemetryRecord =
       project: string
       task: string
       command?: string
+      /**
+       * The tasks with a command this one waits on: its direct dependencies,
+       * a group's seen through to the tasks with a command behind it.
+       * Additive.
+       */
+      dependsOn?: readonly string[]
       ts: number
     }
   | {
@@ -227,6 +245,17 @@ export type TelemetryRecord =
       chunk: string
       ts: number
     }
+  | {
+      v: number
+      kind: 'task.sample'
+      runId: string
+      taskId: string
+      ts: number
+      /** CPU time of the task's live process tree so far, in ms. */
+      cpuMs: number
+      /** Resident memory of the task's live process tree, in bytes. */
+      rssBytes: number
+    }
   | ({ v: number; kind: 'task.end'; runId: string; ts: number } & TaskTelemetry)
   | { v: number; kind: 'run.end'; runId: string; ts: number }
 
@@ -264,6 +293,19 @@ export interface RunSummaryRecord {
   restoredRemoteCount: number
   exitOk: boolean
   tasks: readonly TaskTelemetry[]
+  /**
+   * The run's stages as `VX_TIMING` marks them (startup, load configs,
+   * classify + probe, run graph, …), each a wall window in epoch ms. The
+   * stages ahead of the run lock end before `startedAt`. Additive.
+   */
+  stages?: readonly RunStage[]
+}
+
+/** One stage of a run, a wall window in epoch ms. */
+export interface RunStage {
+  name: string
+  startedAt: number
+  endedAt: number
 }
 
 /**
@@ -286,6 +328,7 @@ export function assembleRunSummary(
     totalDurationMs: number
     exitOk: boolean
     abortedCount: number
+    stages?: readonly RunStage[]
   },
 ): RunSummaryRecord {
   let failedCount = 0
@@ -320,6 +363,7 @@ export function assembleRunSummary(
     restoredRemoteCount,
     exitOk: timing.exitOk,
     tasks,
+    ...(timing.stages !== undefined && timing.stages.length > 0 ? { stages: timing.stages } : {}),
   }
 }
 
@@ -370,8 +414,17 @@ export interface TelemetrySource {
   emitSummary(summary: RunSummaryRecord): void
   /** Await every sink's `flush()` (each crash-isolated, all time-bounded). */
   flush(): Promise<void>
+  /**
+   * Sample this task's process tree every `SAMPLE_MS` until the returned
+   * function is called, the root exits, or the run ends. Present only when
+   * a sink wants `task.sample`: nobody asking costs no timer and no read.
+   */
+  readonly track?: (taskId: string, pid: number) => () => void
 }
 
+/** How often a running task's process tree is sampled. */
+const SAMPLE_MS = 1000
+
 const DEFAULT_KINDS: ReadonlyArray<TelemetryRecord['kind']> = [
   'run.start',
   'task.start',
@@ -397,6 +450,8 @@ export function createTelemetrySource(args: {
   warn?: (message: string) => void
   /** What a sink with no `name` of its own is called: its plugin's (the host's map). */
   owners?: ReadonlyMap<TelemetrySink, string>
+  /** The run's task graph, to see a group dependency through to the tasks behind it. */
+  nodes?: ReadonlyMap<string, TaskNode>
 }): TelemetrySource {
   const { sinks, run, warn } = args
   const runId = run.runId
@@ -426,6 +481,62 @@ export function createTelemetrySource(args: {
   // Precompute which sinks want each kind, so per-event fan-out is a plain
   // array walk with no per-record `wants` scanning.
   const wantsLog = sinks.some((s) => (s.wants ?? DEFAULT_KINDS).includes('task.log'))
+  const wantsSample = sinks.some((s) => (s.wants ?? DEFAULT_KINDS).includes('task.sample'))
+
+  // taskId → the pid of its running root. One timer for every task, alive
+  // only while one runs; a tick still reading when the next is due is not
+  // doubled.
+  const tracked = new Map<string, number>()
+  let timer: ReturnType<typeof setInterval> | undefined
+  let sampling = false
+  let ended = false
+  const stopTimer = (): void => {
+    if (timer !== undefined) clearInterval(timer)
+    timer = undefined
+  }
+  const untrack = (taskId: string, pid: number): void => {
+    if (tracked.get(taskId) === pid) tracked.delete(taskId)
+    if (tracked.size === 0) stopTimer()
+  }
+  const tick = async (): Promise<void> => {
+    if (sampling) return
+    sampling = true
+    try {
+      const entries = [...tracked]
+      const usage = await sampleTrees(entries.map(([, pid]) => pid))
+      const ts = Date.now()
+      for (const [taskId, pid] of entries) {
+        if (ended || tracked.get(taskId) !== pid) continue
+        const u = usage.get(pid)
+        if (u === undefined) {
+          untrack(taskId, pid)
+          continue
+        }
+        deliver({
+          v: TELEMETRY_SCHEMA_VERSION,
+          kind: 'task.sample',
+          runId,
+          taskId,
+          ts,
+          cpuMs: u.cpuMs,
+          rssBytes: u.rssBytes,
+        })
+      }
+    } catch {
+      // A failed look is a missing point, never a broken run.
+    } finally {
+      sampling = false
+    }
+  }
+  const track = (taskId: string, pid: number): (() => void) => {
+    if (ended) return () => {}
+    tracked.set(taskId, pid)
+    if (timer === undefined) {
+      timer = setInterval(() => void tick(), SAMPLE_MS)
+      timer.unref()
+    }
+    return () => untrack(taskId, pid)
+  }
 
   function deliver(record: TelemetryRecord): void {
     for (const sink of sinks) {
@@ -470,6 +581,8 @@ export function createTelemetrySource(args: {
         }
         if (node.config.exec.command !== undefined)
           rec.command = maskedCommand(node.config.exec.command, node.config.exec.env)
+        const deps = commandDeps(node, args.nodes)
+        if (deps.length > 0) rec.dependsOn = deps
         deliver(rec)
         return
       }
@@ -502,6 +615,9 @@ export function createTelemetrySource(args: {
       case 'run:status':
         return // status lines are terminal-rendering noise, not telemetry
       case 'run:end':
+        ended = true
+        tracked.clear()
+        stopTimer()
         deliver({ v: TELEMETRY_SCHEMA_VERSION, kind: 'run.end', runId, ts })
         return
     }
@@ -509,6 +625,7 @@ export function createTelemetrySource(args: {
 
   return {
     subscriber,
+    ...(wantsSample ? { track } : {}),
     emitSummary(summary: RunSummaryRecord): void {
       for (const sink of sinks) {
         if (disabled.has(sink) || sink.onRunSummary === undefined) continue
@@ -556,6 +673,23 @@ export function createTelemetrySource(args: {
   }
 }
 
+/** `node`'s dependencies with a command; a group is seen through, once each. */
+function commandDeps(node: TaskNode, nodes: ReadonlyMap<string, TaskNode> | undefined): string[] {
+  if (nodes === undefined) return node.deps
+  const out: string[] = []
+  const seen = new Set<string>()
+  const stack = [...node.deps].reverse()
+  while (stack.length > 0) {
+    const id = stack.pop()!
+    if (seen.has(id)) continue
+    seen.add(id)
+    const dep = nodes.get(id)
+    if (dep !== undefined && isGroupTask(dep)) stack.push(...[...dep.deps].reverse())
+    else out.push(id)
+  }
+  return out
+}
+
 /**
  * The one projection of an outcome into `TaskTelemetry`, for the streaming
  * `task.end` record and the summary's `tasks[]` alike. Two copies drifted:
@@ -583,6 +717,7 @@ export function taskTelemetryOf(o: TaskOutcome): TaskTelemetry {
   if (o.timedOut === true) t.timedOut = true
   if (o.sandboxViolations !== undefined) t.sandboxViolations = o.sandboxViolations
   if (o.notReady !== undefined) t.notReady = o.notReady
+  if (o.failedAttempts !== undefined) t.failedAttempts = o.failedAttempts
   if (isCacheHit(o.status)) t.restored = o.restored === true
   if (o.wallclockStartNs !== undefined) t.wallclockStartNs = o.wallclockStartNs.toString()
   if (o.wallclockEndNs !== undefined) t.wallclockEndNs = o.wallclockEndNs.toString()

package/src/util/index.ts

@@ -18,7 +18,15 @@ export {
   OUT_OF_FDS_HINT,
 } from './errors.js'
 export { xxh3, xxh3hex } from './hash.js'
-export { mark, printTimings, restartTimings, span } from './timing.js'
+export {
+  beginRun,
+  mark,
+  printTimings,
+  restartTimings,
+  span,
+  stageTimes,
+  type StageTime,
+} from './timing.js'
 export { clampInt, formatElapsed, MAX_TIMEOUT_MS, parseDecimalInt } from './num.js'
 export {
   asTrees,

package/src/util/timing.ts

@@ -2,20 +2,55 @@
 //
 // The CPU profiler attributes a tight loop's cost unreliably and hides
 // where an `await` waited; a stage table answers "where did the warm run
-// go?" directly, and costs one boolean check per mark when off. Marks are
-// cumulative from this module's load — process start and the imports ahead
-// of it sit outside the table; the table shows each stage's own share.
+// go?" directly. Marks are cumulative from this module's load — process
+// start and the imports ahead of it sit outside the table; the table shows
+// each stage's own share. Stage marks are kept whether or not the table
+// prints (a push per stage, ~15 a run): a telemetry sink draws them as
+// spans (`stageTimes`). The per-call `span`s stay off unless enabled.
 
 const enabled = process.env.VX_TIMING !== undefined && process.env.VX_TIMING !== ''
 let t0 = Bun.nanoseconds()
+// The wall clock at t0, read then: `performance.timeOrigin` plus the
+// monotonic clock drifted 34 ms from `Date.now()` on macOS CI, so stages
+// drawn from it sat outside the run they belong to.
+let wall0 = Date.now()
 const marks: Array<[label: string, ns: number]> = []
 
-/** Record the end of a stage. No-op unless `VX_TIMING` is set. */
+let begun = false
+
+/**
+ * A run begins. The process's first counts from this module's load, so
+ * `startup` holds the imports; a later one in the same process (an
+ * embedder's second `run()`) starts a table of its own, or its stages
+ * would follow the last run's and its `startup` would hold the idle between.
+ */
+export function beginRun(): void {
+  if (begun) restartTimings()
+  begun = true
+}
+
+/** Record the end of a stage. */
 export function mark(label: string): void {
-  if (!enabled) return
   marks.push([label, Bun.nanoseconds() - t0])
 }
 
+/** One stage's wall window, in epoch ms. */
+export interface StageTime {
+  name: string
+  startedAt: number
+  endedAt: number
+}
+
+/** The stages marked so far, each from the previous mark's end (the first from this module's load). */
+export function stageTimes(): StageTime[] {
+  let prev = 0
+  return marks.map(([name, ns]) => {
+    const stage = { name, startedAt: wall0 + prev / 1e6, endedAt: wall0 + ns / 1e6 }
+    prev = ns
+    return stage
+  })
+}
+
 const spans = new Map<string, [count: number, ns: number]>()
 const noop = (): void => {}
 
@@ -25,8 +60,8 @@ const noop = (): void => {}
  * its first stage counted the idle wait before it.
  */
 export function restartTimings(): void {
-  if (!enabled) return
   t0 = Bun.nanoseconds()
+  wall0 = Date.now()
   marks.length = 0
   spans.clear()
 }