@vzn/vx 0.0.21 → 0.0.23

package/src/orchestrator/summary.ts

@@ -250,10 +250,12 @@ export function formatSummarySection(
     info.push(context.remoteCacheEnabled ? 'local + remote cache' : 'local cache')
     // The plugin's hand on the run: a task a policy held with a worker
     // free would otherwise have started; without the line nothing says
-    // the policy acted.
+    // the policy acted. The duration is the waits summed, task-seconds
+    // — 85 held tasks read `4783s` beside a 160 s run on TanStack/router
+    // (2026-09-15), so the line says it is a sum.
     if (stats.held !== undefined && stats.held.count > 0) {
       info.push(
-        `admit held ${stats.held.count} task${stats.held.count === 1 ? '' : 's'} ${formatDuration(stats.held.sumMs)}`,
+        `admit held ${stats.held.count} task${stats.held.count === 1 ? '' : 's'}, ${formatDuration(stats.held.sumMs)} in all`,
       )
     }
     lines.push('', row('info', join(info)), row('time', `${formatDuration(totalMs)}${spread}`))
@@ -343,6 +345,51 @@ export function formatAbortedSection(outcomes: readonly TaskOutcome[]): string[]
   return lines
 }
 
+/**
+ * Post-summary section naming the tasks that never started, under the
+ * failure that blocked them. The footer's "1 skipped" says a task was
+ * blocked and nothing about which task or by what; the broad flow prints
+ * no row for a skipped task on purpose (a wide red run would drown in
+ * them), so this is where the reader learns what the failure cost
+ * (item 266). The scheduler records the root of each block on the outcome
+ * (`blockedBy`, item 267); a skip without one is fail-fast's. Empty when
+ * nothing was skipped.
+ */
+export function formatSkippedSection(outcomes: readonly TaskOutcome[]): string[] {
+  // A group (no `exec`) never starts by definition and is not a task the
+  // tally counts; listing a blocked one here named three "tasks" the legend
+  // beside it counted as none (item 281). Its members' rows say enough.
+  const skipped = outcomes.filter((o) => o.status === 'skipped' && !isGroupTask(o.node))
+  if (skipped.length === 0) return []
+  const byId = new Map(outcomes.map((o) => [o.node.id, o]))
+  const causeOf = (o: TaskOutcome): string => {
+    if (o.blockedBy === undefined) return 'after the run stopped (fail-fast)'
+    const root = byId.get(o.blockedBy)
+    return root?.status === 'aborted'
+      ? `after ${o.blockedBy} was aborted`
+      : `after ${o.blockedBy} failed`
+  }
+  const groups = new Map<string, string[]>()
+  for (const o of skipped) {
+    const cause = causeOf(o)
+    const list = groups.get(cause) ?? []
+    list.push(o.node.id)
+    groups.set(cause, list)
+  }
+  const lines = [
+    '',
+    `  Skipped:  ${skipped.length} task${skipped.length === 1 ? '' : 's'} never started \u2014 blocked upstream`,
+  ]
+  const NAMES = 8
+  for (const [cause, ids] of [...groups].sort((a, b) => b[1].length - a[1].length)) {
+    ids.sort()
+    const shown = ids.slice(0, NAMES).join(', ')
+    const more = ids.length > NAMES ? ` \u2026 +${ids.length - NAMES} more` : ''
+    lines.push(`    \u2298 ${cause}: ${shown}${more}`)
+  }
+  return lines
+}
+
 /**
  * Post-summary section naming the tasks this run proved flaky: identical
  * inputs, both outcomes on record (or a retry within the run). A red run

package/src/orchestrator/task-hash.ts

@@ -1,9 +1,11 @@
+import { lstatSync } from 'node:fs'
 import path from 'node:path'
 import type { TaskConfig, CacheConfig } from '../config.js'
-import type { WorkspaceFilesCache } from '../cache/index.js'
+import type { ProjectFilesCache, WorkspaceFilesCache } from '../cache/index.js'
 import {
   type CacheKeyInput,
   type CacheLayer,
+  FILE_HASH_RACY_MS,
   resolveInputs,
   type GitFilesCache,
   WORKSPACE_OUTPUT_PREFIX,
@@ -20,6 +22,9 @@ import { expandGroupUpstream, filterUpstreamHashes } from './upstream.js'
  *     project re-reads the same `package.json`; without this, a
  *     monorepo with N projects × M tasks per project does N×M reads
  *     of the same bytes.
+ *   - `projectFiles`: keyed by the project and its input DECLARATION.
+ *     Tasks of one project that declare the same inputs and outputs
+ *     resolve the same file list; see `ProjectFilesCache`.
  *   - `taskConfig`: keyed by the resolved-config object reference.
  *     Each task's config is created once at prepareRun time; the
  *     JSON.stringify + xxh3 of it is deterministic. WeakMap so
@@ -34,6 +39,7 @@ export interface HashCache {
   runtime: Map<string, Promise<string>>
   workspaceRuntime: Map<string, Promise<string>>
   workspaceFiles: WorkspaceFilesCache
+  projectFiles: ProjectFilesCache
 }
 
 export function createHashCache(): HashCache {
@@ -43,6 +49,7 @@ export function createHashCache(): HashCache {
     runtime: new Map(),
     workspaceRuntime: new Map(),
     workspaceFiles: new Map(),
+    projectFiles: new Map(),
   }
 }
 
@@ -101,6 +108,18 @@ export async function computeTaskHash(args: ComputeHashArgs): Promise<string> {
   }
 }
 
+/**
+ * One file the key folded: the digest it folded, and since when (ms since
+ * the epoch) that digest is known to be the file's content — the
+ * enumeration's start for an index OID (`git status` vouched for it then),
+ * the describe's start for a digest hashed from the file.
+ */
+export interface InputFact {
+  path: string
+  digest: string
+  since: number
+}
+
 /**
  * The key AND the structured input set behind it, for the executor seam.
  * Miss path only: it re-runs the (memoized) resolution `computeTaskHash`
@@ -112,7 +131,8 @@ export async function computeTaskHash(args: ComputeHashArgs): Promise<string> {
  */
 export async function describeTaskInputs(
   args: ComputeHashArgs,
-): Promise<{ hash: string; inputs: TaskInputs }> {
+): Promise<{ hash: string; inputs: TaskInputs; facts: InputFact[]; describedAt: number }> {
+  const hashedAt = Date.now()
   const input = await resolveKeyInput(args)
   const hash = await args.cache.key(input)
   const sorted = [...input.inputFiles].sort()
@@ -123,6 +143,22 @@ export async function describeTaskInputs(
     path: relPosix(input.workspaceRoot, f),
     digest: digests[i]!,
   }))
+  // The package.json digest is a per-run memo that may predate this
+  // describe, so it is dated from the enumeration, the earliest a run
+  // learns anything.
+  const indexedAt = args.gitFilesCache?.enumeratedAtMs ?? 0
+  const facts: InputFact[] = sorted.map((f, i) => ({
+    path: f,
+    digest: digests[i]!,
+    since: input.fileHashes?.has(f) === true ? indexedAt : hashedAt,
+  }))
+  if (input.projectPackageJsonHash !== '') {
+    facts.push({
+      path: path.join(args.node.projectDir, 'package.json'),
+      digest: input.projectPackageJsonHash,
+      since: indexedAt,
+    })
+  }
   // Groups expanded: this list describes the INPUT ROOT, not the key, and a
   // group produces nothing of its own (see `CacheKeyInput.upstreamGraft`).
   // Sorted by hash so the list — and any action digest derived from it — is
@@ -143,6 +179,8 @@ export async function describeTaskInputs(
   })
   return {
     hash,
+    facts,
+    describedAt: hashedAt,
     inputs: {
       files,
       env: input.envValues.map(([name, value]) => ({ name, value })),
@@ -159,6 +197,41 @@ export async function describeTaskInputs(
   }
 }
 
+/**
+ * The first input whose content may no longer be the digest the key folded
+ * (item 743), or undefined when every one still is. One `lstat` per file:
+ * a file whose ctime is older than its fact (by the racy window, as git
+ * judges its index) has not been written since, and one that is not gets
+ * hashed again and compared. ctime, because no writer can set it back. A
+ * file that is gone has moved. One written at or after `commandFrom` (the
+ * describe just before the command) has moved whatever it holds now: an
+ * input changed and changed BACK while the command ran matched its digest
+ * again, and the entry saved the other content's output under the key (item
+ * 1015). A write before it — an upstream's — is judged by content, as before.
+ */
+export async function movedInput(
+  facts: readonly InputFact[],
+  cache: CacheLayer,
+  commandFrom?: number,
+): Promise<string | undefined> {
+  const suspects: InputFact[] = []
+  for (const f of facts) {
+    let ctimeMs: number
+    try {
+      ctimeMs = lstatSync(f.path).ctimeMs
+    } catch {
+      return f.path
+    }
+    if (commandFrom !== undefined && ctimeMs >= commandFrom) return f.path
+    if (ctimeMs >= f.since - FILE_HASH_RACY_MS) suspects.push(f)
+  }
+  for (const f of suspects) {
+    const now = await cache.hashFile(f.path).catch(() => undefined)
+    if (now !== f.digest) return f.path
+  }
+  return undefined
+}
+
 async function resolveKeyInput(args: ComputeHashArgs): Promise<CacheKeyInput> {
   const cfg = args.node.config
   const cacheCfg: CacheConfig | undefined = cfg.cache
@@ -178,6 +251,7 @@ async function resolveKeyInput(args: ComputeHashArgs): Promise<CacheKeyInput> {
           runtimeCache: args.hashCache.runtime,
           workspaceRuntimeCache: args.hashCache.workspaceRuntime,
           workspaceFilesCache: args.hashCache.workspaceFiles,
+          projectFilesCache: args.hashCache.projectFiles,
         }
       : {}),
   })
@@ -251,8 +325,9 @@ async function resolveKeyInput(args: ComputeHashArgs): Promise<CacheKeyInput> {
  * participate because the loader has already baked them into the
  * resolved object before we serialize.
  *
- * The schema is JSON-serializable by construction (no functions in
- * fields). The `hashCache.taskConfig` WeakMap is consulted first —
+ * A loaded config is JSON data: every load path refuses a value JSON
+ * cannot carry (workspace/json-data.ts, item 701), so this JSON is the
+ * config as vx reads it. The `hashCache.taskConfig` WeakMap is consulted first —
  * each task's config object is created once per run, so a hit there
  * skips the JSON.stringify + xxh3 entirely.
  */
@@ -276,9 +351,14 @@ function hashTaskConfig(cfg: TaskConfig, hashCache?: HashCache): string {
  * laptop from a worker pool over nothing. (`exec.resources` rode the same
  * rule until 2026-09-12, when reservations left the config for the
  * schedule plugin that learns them; a config never declares them now.) A
- * config that declares no `remote` takes the fast path and stringifies
- * byte-identically to before the field existed (why this needs no
- * CACHE_VERSION bump). `timeout`/`retries` stay folded — their keys are
+ * config that declares no `remote` takes the fast path. That is a
+ * SHORTCUT, not the reason the field needed no CACHE_VERSION bump:
+ * measured (item 505), the spread path stringifies byte-identically for
+ * every config that HAS an `exec`, since a spread keeps an existing key
+ * in place. Its one behavioural case is a config with NO `exec`, where
+ * the spread would add `exec: {}` — and nothing reaches here that way,
+ * proven by throwing on a no-exec config and running the whole suite:
+ * a group takes `computeGroupHash` instead. `timeout`/`retries` stay folded — their keys are
  * distinct by design (see the decision log); stripping them retroactively
  * would bump CACHE_VERSION.
  */

package/src/orchestrator/task-log-buffer.ts

@@ -79,13 +79,6 @@ interface InFlight {
   truncatedHeadChars: number
 }
 
-/**
- * Bounded per-run capture. `append` keeps a chunk LIST + running char count per
- * task, evicting whole chunks from the head past `TASK_LOG_TAIL_CHARS` — no
- * string concatenation until `drain`, so a cache-hit replay (one big chunk) is
- * one array push, zero copies. `finish` decides retention; `drain` emits the
- * bundle, failures first.
- */
 /**
  * Per-retained-chunk overhead, in char-equivalents, charged against the RUN
  * budget on top of the characters themselves.
@@ -110,6 +103,13 @@ function budgetCost(chars: number, chunkCount: number): number {
   return chars + chunkCount * CHUNK_OVERHEAD_CHARS
 }
 
+/**
+ * Bounded per-run capture. `append` keeps a chunk LIST + running char count per
+ * task, evicting whole chunks from the head past `TASK_LOG_TAIL_CHARS` — no
+ * string concatenation until `drain`, so a cache-hit replay (one big chunk) is
+ * one array push, zero copies. `finish` decides retention; `drain` emits the
+ * bundle, failures first.
+ */
 export class TaskLogBuffer {
   private readonly inFlight = new Map<string, InFlight>()
   private readonly retained = new Map<string, Retained>()

package/src/orchestrator/telemetry-host.ts

@@ -18,6 +18,7 @@ import type {
   TelemetryContext,
   TelemetrySink,
 } from './telemetry.js'
+import { settleWithin, teardownTimeoutMs } from '../util/index.js'
 import { createTelemetrySource } from './telemetry.js'
 import type { VxPlugin } from './plugin.js'
 
@@ -79,7 +80,18 @@ export async function subscribeTelemetry(
     // check sits INSIDE the try so a throwing `wants` getter is caught too.
     let accepted: TelemetrySink[]
     try {
-      const result = await plugin.telemetry(ctx)
+      // Bounded as flush and teardown are: a hook that never settles held
+      // the run before its first task, forever or until the loop drained
+      // and Bun exited 0 (item 921).
+      let result: Awaited<ReturnType<NonNullable<VxPlugin['telemetry']>>> | undefined
+      const ms = teardownTimeoutMs()
+      const arrived = await settleWithin(
+        Promise.resolve(plugin.telemetry(ctx)).then((r) => {
+          result = r
+        }),
+        ms,
+      )
+      if (!arrived) throw new Error(`did not initialize within ${ms} ms`)
       const list = result === undefined ? [] : Array.isArray(result) ? result : [result]
       accepted = list.map((sink) => checkSink(sink))
     } catch (err) {
@@ -94,15 +106,12 @@ export async function subscribeTelemetry(
   if (sinks.length === 0) return undefined
 
   const source = createTelemetrySource({ sinks, run, warn: (m) => ctx.warn(m) })
+  // The bus's unsubscribe is idempotent, so it is the handle's dispose as
+  // is (a once-flag here survived item 654).
   const dispose = bus.subscribe(source.subscriber)
-  let disposed = false
   return {
     emitSummary: (summary) => source.emitSummary(summary),
     flush: () => source.flush(),
-    dispose() {
-      if (disposed) return
-      disposed = true
-      dispose()
-    },
+    dispose,
   }
 }

package/src/orchestrator/telemetry.ts

@@ -16,12 +16,12 @@
 // what or how tasks run. Contrast `cache`/`executor`, which return objects
 // core calls INTO; those are the behavior capabilities, kept separate.
 
-import type { TaskStatus } from '../graph/index.js'
+import type { TaskOutcome, TaskStatus } from '../graph/index.js'
 import { settleWithin, teardownTimeoutMs } from '../util/index.js'
 import type { RunEvent, RunEventSubscriber } from './events.js'
 
 /** Bumped when the record shape changes. Readers MUST check `v`. */
-export const TELEMETRY_SCHEMA_VERSION = 2
+export const TELEMETRY_SCHEMA_VERSION = 3
 
 /** Where a task's result came from, derived ONCE in core from the status. */
 export type CacheSource = 'miss' | 'local' | 'remote' | 'none'
@@ -75,8 +75,6 @@ const PASSES: Record<TaskStatus, boolean> = {
  */
 export const TASK_STATUSES: readonly TaskStatus[] = Object.keys(PASSES) as TaskStatus[]
 
-const KNOWN_STATUSES: ReadonlySet<string> = new Set(TASK_STATUSES)
-
 /**
  * Did the task pass? A cache hit counts — it produced the same result without
  * spending the time, which is the whole point. `skipped` and `aborted` do NOT:
@@ -97,7 +95,8 @@ export function isPassStatus(status: string): boolean {
  * cannot disagree about what a hit is. Unknown strings read as not-a-hit.
  */
 export function isCacheHit(status: string): boolean {
-  if (!KNOWN_STATUSES.has(status)) return false
+  // No known-status guard: an unknown string falls through the switch to
+  // undefined, which is neither hit source (the guard survived item 654).
   const source = deriveCacheSource(status as TaskStatus)
   return source === 'local' || source === 'remote'
 }
@@ -169,6 +168,18 @@ export interface TaskTelemetry {
    *  / `--retry` produced more than one attempt. A retried-then-passed task is
    *  flaky by definition; this is the telemetry-side flaky signal. */
   attempts?: number
+  /**
+   * On a `skipped` task: the id of the failed (or aborted) task at the
+   * root of what blocked it, through any chain of skips. Absent on a
+   * fail-fast skip and on every other status. Additive (schema stays 2).
+   */
+  blockedBy?: string
+  /** On a `failed` task: vx's own `timeout` killed it (exit 143 is the deadline, not a signal). Additive. */
+  timedOut?: true
+  /** On a sandboxed task: how many declared-boundary violations the sandbox recorded (a failure on its own). Additive. */
+  sandboxViolations?: number
+  /** On a failed persistent task: why it never became ready. `exitCode` is the child's own when it exited. Additive. */
+  notReady?: 'timeout' | 'exited' | 'spawn'
   /** bigint hrtime ns relative to run t=0, encoded as a decimal string. */
   wallclockStartNs?: string
   wallclockEndNs?: string
@@ -227,6 +238,12 @@ export interface RunSummaryRecord {
   totalDurationMs: number
   taskCount: number
   failedCount: number
+  /**
+   * Tasks a shutdown signal or an embedder's abort killed (v3). They are
+   * not in `tasks`, which holds real runs only, so without this a stopped
+   * run read as a failure with nothing failed (item 851).
+   */
+  abortedCount: number
   hitCount: number
   hitLocalCount: number
   hitRemoteCount: number
@@ -248,7 +265,13 @@ export interface RunSummaryRecord {
 export function assembleRunSummary(
   run: RunContextRecord,
   tasks: readonly TaskTelemetry[],
-  timing: { startedAt: number; endedAt: number; totalDurationMs: number; exitOk: boolean },
+  timing: {
+    startedAt: number
+    endedAt: number
+    totalDurationMs: number
+    exitOk: boolean
+    abortedCount: number
+  },
 ): RunSummaryRecord {
   let failedCount = 0
   let hitLocalCount = 0
@@ -266,6 +289,7 @@ export function assembleRunSummary(
     totalDurationMs: timing.totalDurationMs,
     taskCount: tasks.length,
     failedCount,
+    abortedCount: timing.abortedCount,
     hitCount: hitLocalCount + hitRemoteCount,
     hitLocalCount,
     hitRemoteCount,
@@ -352,8 +376,9 @@ export function createTelemetrySource(args: {
   // Never silently. The standing rule is that a never-fail path must still
   // WARN — telemetry that vanishes without a word is indistinguishable from
   // telemetry nobody configured.
+  // Every hook call site skips a disabled sink first, so this runs at most
+  // once per sink without a guard of its own (item 654).
   const disable = (sink: TelemetrySink, hook: string, err: unknown): void => {
-    if (disabled.has(sink)) return
     disabled.add(sink)
     warn?.(
       `[vx] telemetry sink '${sink.name}' threw in ${hook}; disabled for this run: ${err instanceof Error ? err.message : String(err)}`,
@@ -423,30 +448,13 @@ export function createTelemetrySource(args: {
       case 'task:complete': {
         const { node, outcome } = event
         if (node.config.exec === undefined) return // group task
-        const rec: TelemetryRecord = {
+        deliver({
           v: TELEMETRY_SCHEMA_VERSION,
           kind: 'task.end',
           runId,
           ts,
-          taskId: node.id,
-          project: node.projectName,
-          task: node.taskName,
-          status: outcome.status,
-          cacheSource: deriveCacheSource(outcome.status),
-          exitCode: outcome.exitCode,
-          durationMs: outcome.durationMs,
-        }
-        if (outcome.hash !== undefined) rec.hash = outcome.hash
-        if (outcome.cpuMs !== undefined) rec.cpuMs = outcome.cpuMs
-        if (outcome.peakRssBytes !== undefined) rec.peakRssBytes = outcome.peakRssBytes
-        if (outcome.where !== undefined) rec.where = outcome.where
-        if (outcome.outputs !== undefined) rec.outputs = outcome.outputs
-        if (outcome.attempts !== undefined) rec.attempts = outcome.attempts
-        if (outcome.wallclockStartNs !== undefined)
-          rec.wallclockStartNs = outcome.wallclockStartNs.toString()
-        if (outcome.wallclockEndNs !== undefined)
-          rec.wallclockEndNs = outcome.wallclockEndNs.toString()
-        deliver(rec)
+          ...taskTelemetryOf(outcome),
+        })
         return
       }
       case 'run:status':
@@ -476,7 +484,8 @@ export function createTelemetrySource(args: {
       const ms = teardownTimeoutMs()
       // Bounded, for the same reason plugin teardown is bounded in
       // plugin-host.ts: run() awaits this BEFORE closeCache() and before it
-      // returns, and bin.ts is `process.exit(await run(...))`. A sink whose
+      // returns, and bin.ts exits with run()'s code once it returns (through
+      // stdout's end callback since 2026-09-15; the code is still that). A sink whose
       // flush never settles therefore drains the event loop with no exit code
       // pending — Bun exits 0 and a FAILED run reports green, with the cache's
       // accessed_at bumps and every later plugin's teardown lost with it.
@@ -502,3 +511,35 @@ export function createTelemetrySource(args: {
     },
   }
 }
+
+/**
+ * The one projection of an outcome into `TaskTelemetry`, for the streaming
+ * `task.end` record and the summary's `tasks[]` alike. Two copies drifted:
+ * `task.end` dropped `blockedBy`, `timedOut`, `sandboxViolations` and
+ * `notReady`, so a streaming sink saw a timed-out or blocked task as a plain
+ * failure or skip (item 660).
+ */
+export function taskTelemetryOf(o: TaskOutcome): TaskTelemetry {
+  const t: TaskTelemetry = {
+    taskId: o.node.id,
+    project: o.node.projectName,
+    task: o.node.taskName,
+    status: o.status,
+    cacheSource: deriveCacheSource(o.status),
+    exitCode: o.exitCode,
+    durationMs: o.durationMs,
+  }
+  if (o.hash !== undefined) t.hash = o.hash
+  if (o.cpuMs !== undefined) t.cpuMs = o.cpuMs
+  if (o.peakRssBytes !== undefined) t.peakRssBytes = o.peakRssBytes
+  if (o.where !== undefined) t.where = o.where
+  if (o.outputs !== undefined) t.outputs = o.outputs
+  if (o.attempts !== undefined) t.attempts = o.attempts
+  if (o.blockedBy !== undefined) t.blockedBy = o.blockedBy
+  if (o.timedOut === true) t.timedOut = true
+  if (o.sandboxViolations !== undefined) t.sandboxViolations = o.sandboxViolations
+  if (o.notReady !== undefined) t.notReady = o.notReady
+  if (o.wallclockStartNs !== undefined) t.wallclockStartNs = o.wallclockStartNs.toString()
+  if (o.wallclockEndNs !== undefined) t.wallclockEndNs = o.wallclockEndNs.toString()
+  return t
+}