@vzn/vx 0.0.651 → 0.0.653

package/src/orchestrator/telemetry.ts

@@ -10,13 +10,15 @@
 // rendering-oriented `WireEvent` stream.
 //
 // A sink is observe-only BY CONSTRUCTION: its only input is a plain-data
-// record (one object every sink receives, not frozen); its `TelemetryContext` carries read-only metadata and NO mutable
+// record (one object every sink receives, frozen once, deep: a sink that
+// cleared a `task.end`'s `failedAttempts` cleared the outcome's own and
+// crashed the run's failure log); its `TelemetryContext` carries read-only metadata and NO mutable
 // run handle (no bus, no Cache, no RunRequest). There is no API path from a
 // sink back into scheduling/caching/exec — telemetry provably cannot change
 // what or how tasks run. Contrast `cache`/`executor`, which return objects
 // core calls INTO; those are the behavior capabilities, kept separate.
 
-import { sampleTrees, type MachineReport } from '../exec/index.js'
+import { sampleTrees, type MachineReport, type WorkerAdvice } from '../exec/index.js'
 import {
   type CacheSource,
   deriveCacheSource,
@@ -34,7 +36,8 @@ import {
   teardownTimeoutMs,
   thrownMessage,
 } from '../util/index.js'
-import type { RunEvent, RunEventSubscriber } from '../report/index.js'
+import { neverStarted, type RunEvent, type RunEventSubscriber } from '../report/index.js'
+import { freezeDeep } from './read-only.js'
 
 /** Bumped when the record shape changes. Readers MUST check `v`. */
 export const TELEMETRY_SCHEMA_VERSION = 4
@@ -256,14 +259,20 @@ export interface RunSummaryRecord {
   startedAt: number
   endedAt: number
   totalDurationMs: number
+  /** Every task the run had, as the terminal's total: `tasks` plus `notRunCount`. */
   taskCount: number
   failedCount: number
   /**
-   * Tasks a shutdown signal or an embedder's abort killed (v3). They are
-   * not in `tasks`, which holds real runs only, so without this a stopped
-   * run read as a failure with nothing failed (item 851).
+   * Tasks a shutdown signal or an embedder's abort killed once started (v3),
+   * each in `tasks` as `aborted`: without the count a stopped run read as a
+   * failure with nothing failed (item 851).
    */
   abortedCount: number
+  /**
+   * Tasks the stop reached before they started: the terminal's `not run`.
+   * Not in `tasks`, since they never ran. Absent when none. Additive.
+   */
+  notRunCount?: number
   hitCount: number
   /** Hits by the layer that answered, up-to-date ones included. */
   hitLocalCount: number
@@ -301,6 +310,8 @@ export interface RunSummaryRecord {
    * (`TaskExecutor.idealMs`), beside `totalDurationMs`. Additive.
    */
   idealMs?: number
+  /** The workers that would have run it best (`TaskExecutor.advice`). Additive. */
+  advice?: readonly WorkerAdvice[]
 }
 
 /** One stage of a run, a wall window in epoch ms. */
@@ -316,8 +327,9 @@ export interface RunStage {
  * computed: both a local `run()` and the distributed controller build their
  * `TaskTelemetry[]` (each via `deriveCacheSource`) and call this, so a
  * distributed run and a local run produce byte-identical summaries and land in
- * the same ingest. The per-task tallies (taskCount / failedCount /
- * hitLocal|Remote|Count) derive from `tasks`; `totalDurationMs` (wall time) and
+ * the same ingest. The per-task tallies (failedCount /
+ * hitLocal|Remote|Count) derive from `tasks`, and `taskCount` from `tasks`
+ * plus `notRunCount`; `totalDurationMs` (wall time) and
  * `exitOk` (the run's overall verdict — which counts skipped tasks beyond the
  * recorded task list) are run-level facts and are passed in.
  */
@@ -330,10 +342,12 @@ export function assembleRunSummary(
     totalDurationMs: number
     exitOk: boolean
     abortedCount: number
+    notRunCount?: number
     stages?: readonly RunStage[]
     uploads?: { count: number; bytes: number; ms: number; failed: number }
     machines?: readonly MachineReport[]
     idealMs?: number | undefined
+    advice?: readonly WorkerAdvice[]
   },
 ): RunSummaryRecord {
   let failedCount = 0
@@ -357,9 +371,14 @@ export function assembleRunSummary(
     startedAt: timing.startedAt,
     endedAt: timing.endedAt,
     totalDurationMs: timing.totalDurationMs,
-    taskCount: tasks.length,
+    // The terminal's total: the tasks a stop reached first are not in
+    // `tasks`, yet a summary of 2 beside a terminal's 3 read as a lost task.
+    taskCount: tasks.length + (timing.notRunCount ?? 0),
     failedCount,
     abortedCount: timing.abortedCount,
+    ...(timing.notRunCount !== undefined && timing.notRunCount > 0
+      ? { notRunCount: timing.notRunCount }
+      : {}),
     hitCount: hitLocalCount + hitRemoteCount,
     hitLocalCount,
     hitRemoteCount,
@@ -376,10 +395,15 @@ export function assembleRunSummary(
       ? { machines: timing.machines }
       : {}),
     ...(timing.idealMs !== undefined ? { idealMs: timing.idealMs } : {}),
+    ...(timing.advice !== undefined && timing.advice.length > 0 ? { advice: timing.advice } : {}),
   }
 }
 
-/** A telemetry consumer. Observe-only: receives records, holds no run handle. */
+/**
+ * A telemetry consumer. Observe-only: receives records, frozen, holds no
+ * run handle. A write to a record throws, and the sink is switched off for
+ * the run as for any throw.
+ */
 export interface TelemetrySink {
   readonly name?: string
   /**
@@ -391,7 +415,11 @@ export interface TelemetrySink {
   readonly wants?: ReadonlyArray<TelemetryRecord['kind']>
   /** A streaming record. MUST return promptly — buffer; do NOT await I/O. */
   onRecord?(record: TelemetryRecord): void
-  /** The per-run summary, at run:end. MUST return promptly. */
+  /**
+   * The per-run summary, once at run:end; a requested persistent task the
+   * foreground keeps sends it when that task exits, so it reads the run's
+   * final verdict. MUST return promptly.
+   */
   onRunSummary?(summary: RunSummaryRecord): void
   /**
    * Drain buffered data. Awaited at run:end under a deadline — a sink that
@@ -429,6 +457,12 @@ export interface TelemetrySource {
    * each as the step `<plugin>:telemetry` through `step`.
    */
   flush(step?: <T>(id: string, work: () => Promise<T>) => Promise<T>): Promise<void>
+  /**
+   * Send the held `task.end` of each persistent task not in `keep`, read
+   * from `final` (by id): a server's outcome can change after its ready,
+   * when it dies before the run stops it or ends a kept session.
+   */
+  endServers(final: readonly TaskOutcome[], keep?: ReadonlySet<string>): void
   /**
    * Sample this task's process tree every `SAMPLE_MS` until the returned
    * function is called, the root exits, or the run ends. Present only when
@@ -598,6 +632,9 @@ export function createTelemetrySource(args: {
   }
 
   function deliver(record: TelemetryRecord): void {
+    // Once, for every sink: a `task.end` holds the outcome's own arrays, and
+    // a sink that cleared `failedAttempts` crashed the run's failure log.
+    freezeDeep(record)
     for (const sink of sinks) {
       if (disabled.has(sink) || sink.onRecord === undefined) continue
       const kinds = sink.wants ?? DEFAULT_KINDS
@@ -663,6 +700,16 @@ export function createTelemetrySource(args: {
       case 'task:complete': {
         const { node, outcome } = event
         if (node.config.exec === undefined) return // group task
+        // The stop reached it first: no task.start, and the summary counts
+        // it in `notRunCount`, not `tasks`, so no task.end either.
+        if (neverStarted(outcome)) return
+        // A server's end waits for its final outcome (`endServers`): one
+        // task.end per task, and the one sent at ready read `success` over
+        // a crash after it.
+        if (node.config.exec.persistent !== undefined) {
+          heldEnds.set(node.id, { ts, outcome })
+          return
+        }
         deliver({
           v: TELEMETRY_SCHEMA_VERSION,
           kind: 'task.end',
@@ -683,10 +730,26 @@ export function createTelemetrySource(args: {
     }
   }
 
+  const heldEnds = new Map<string, { ts: number; outcome: TaskOutcome }>()
   return {
     subscriber,
     ...(wantsSample ? { track } : {}),
+    endServers(final, keep) {
+      for (const [id, held] of heldEnds) {
+        if (keep?.has(id) === true) continue
+        heldEnds.delete(id)
+        const outcome = final.find((o) => o.node.id === id) ?? held.outcome
+        deliver({
+          v: TELEMETRY_SCHEMA_VERSION,
+          kind: 'task.end',
+          runId,
+          ts: held.ts,
+          ...taskTelemetryOf(outcome),
+        })
+      }
+    },
     emitSummary(summary: RunSummaryRecord): void {
+      freezeDeep(summary)
       for (const sink of sinks) {
         if (disabled.has(sink) || sink.onRunSummary === undefined) continue
         try {
@@ -708,9 +771,12 @@ export function createTelemetrySource(args: {
       // accessed_at bumps and every later plugin's teardown lost with it.
       // Sinks race concurrently, so each still gets the whole budget.
       const deadline = new AbortController()
+      // The deadline's warning names the sinks still out, as a throw names its own.
+      const pending = new Set<TelemetrySink>()
       const all = Promise.all(
         sinks.map(async (sink) => {
           if (disabled.has(sink) || sink.flush === undefined) return
+          pending.add(sink)
           try {
             await step(`${args.owners?.get(sink) ?? label(sink)}:telemetry`, () =>
               sink.flush!(deadline.signal),
@@ -719,12 +785,17 @@ export function createTelemetrySource(args: {
             // A flush failure can never break the run — but it still costs
             // this sink its export, so say so rather than dropping it.
             warn?.(`[vx] telemetry sink '${label(sink)}' failed to flush: ${thrownMessage(err)}`)
+          } finally {
+            pending.delete(sink)
           }
         }),
       )
       if (!(await settleWithin(all, ms))) {
         deadline.abort(new Error(`telemetry flush deadline (${ms}ms)`))
-        warn?.(`[vx] telemetry flush timed out after ${ms}ms; buffered records lost`)
+        const names = [...pending].map((s) => `'${label(s)}'`).join(', ')
+        warn?.(
+          `[vx] telemetry flush timed out after ${ms}ms (sink${pending.size === 1 ? '' : 's'} ${names}); buffered records lost`,
+        )
         // A sink told of the abort says what it lost as its request ends; the
         // run ended first and that line was lost (vx-otel, on CI). Briefly.
         await settleWithin(all, ABORT_GRACE_MS)
@@ -763,7 +834,8 @@ export function taskTelemetryOf(o: TaskOutcome): TaskTelemetry {
     project: o.node.projectName,
     task: o.node.taskName,
     status: o.status,
-    cacheSource: deriveCacheSource(o.status),
+    // A hit whose restore failed is in no cache column: neither a miss nor a hit.
+    cacheSource: o.restoreFailed === true ? 'none' : deriveCacheSource(o.status),
     exitCode: o.exitCode,
     durationMs: o.durationMs,
   }

package/src/orchestrator/upstream.ts

@@ -1,5 +1,6 @@
 import {
   selectFoldedDeps,
+  type StandIns,
   type FoldCandidate,
   type TaskNode,
   type TaskOutcome,
@@ -72,10 +73,11 @@ export function filterUpstreamHashes(
   filter: readonly string[] | undefined,
   selfProjectName: string,
   selfTaskId: string,
+  standIns?: StandIns,
 ): Array<[upstreamTaskId: string, hash: string]> {
   const candidates: FoldCandidate[] = []
   for (const u of upstream) if (u.hash) candidates.push({ node: u.node, unit: u.hash })
-  return selectFoldedDeps(candidates, filter, selfProjectName, selfTaskId).map((c) => [
+  return selectFoldedDeps(candidates, filter, selfProjectName, selfTaskId, standIns).map((c) => [
     c.node.id,
     c.unit,
   ])

package/src/report/events.ts

@@ -8,18 +8,18 @@
 
 import { exitSignal } from '../exec/index.js'
 import type { InputChanges, TaskNode, TaskOutcome } from '../graph/index.js'
-import { isGroupTask } from '../graph/index.js'
+import { isGroupTask, isKeyFailed, markOutputsFailed, outputsFailedOf } from '../graph/index.js'
 import { heldSecrets, maskedCommand, type SecretMask, secretMask } from '../util/index.js'
 import type { Logger } from './logger.js'
 import type { RunContext } from './summary.js'
 
 /**
- * A task no cache answered for: it declares no `cache`, or the run's policy
+ * A task no cache answered for: it has no `cache: true`, or the run's policy
  * read and wrote nothing. Its run is `no-cache`, never a miss, on every
  * surface — `--dry`'s plan, the frame, the footer, the run history.
  */
 export function ranNoCache(o: { node: TaskNode; cacheOff?: boolean }): boolean {
-  return o.node.config.cache === undefined || o.cacheOff === true
+  return o.node.config.cache !== true || o.cacheOff === true
 }
 
 /** Payload of the `run:start` event — mirrors the Logger.runStart hook. */
@@ -41,7 +41,8 @@ export interface RunStartInfo {
  * refs — zero-copy, because the default terminal subscriber runs on the
  * same thread and reads those objects directly. Crossing a thread or RPC
  * needs the serializable WireEvent form (see `projectNode` / `toWireEvent`
- * below); that mapping is the boundary's job, not the producer's.
+ * below); that mapping is the boundary's job, not the producer's. A
+ * plugin gets read-only views of them (orchestrator/read-only.ts).
  */
 export type RunEvent =
   | { kind: 'run:start'; info: RunStartInfo }
@@ -281,8 +282,10 @@ export interface OutcomeView {
    * and counting groups as executed tasks drifts from the terminal.
    */
   isGroup?: boolean
-  /** Executed with no `cache` block: never consulted the cache, not a miss. */
+  /** Executed without `cache: true`: never consulted the cache, not a miss. */
   noCache?: boolean
+  /** A failed hit whose restore threw (see `TaskOutcome`): neither a miss nor a hit. */
+  restoreFailed?: true
   /** Cache hits only: the exec time the entry was stored with — the work
    *  this hit skipped, as opposed to `durationMs`, the restore it cost. */
   storedDurationMs?: number
@@ -344,12 +347,18 @@ export function outcomeWord(o: Pick<OutcomeView, 'status' | 'restored'>): string
   }
 }
 
-/** `outcomeWord` with the exit code a failure carries: `failed (exit 9)`. */
+/**
+ * `outcomeWord` with the exit code a failure carries: `failed (exit 9)`.
+ * A hit whose restore failed ran nothing, so it has no exit to name, nor
+ * does a task whose key could not be derived (`isKeyFailed`) or whose
+ * output clean or save vx refused (`outputsFailedOf`): the step reads.
+ */
 export function outcomeLabel(
   o: Pick<
     OutcomeView,
     | 'status'
     | 'restored'
+    | 'restoreFailed'
     | 'exitCode'
     | 'timedOut'
     | 'sandboxViolations'
@@ -359,6 +368,11 @@ export function outcomeLabel(
     | 'otherPlatform'
   >,
 ): string {
+  if (o.status === 'failed' && o.restoreFailed === true) return 'failed (cache restore failed)'
+  if (o.status === 'failed' && isKeyFailed(o)) return 'failed (cache key not derived)'
+  const step = o.status === 'failed' ? outputsFailedOf(o) : undefined
+  if (step !== undefined)
+    return step === 'save' ? 'failed (cache save failed)' : 'failed (output clean failed)'
   if (o.status === 'failed')
     return failedLabel(o.exitCode, o.timedOut, o.sandboxViolations, o.notReady, o.inputMoved)
   if (o.status === 'skipped') return skippedLabel(o)
@@ -449,6 +463,9 @@ export function projectOutcome(outcome: TaskOutcome): OutcomeView {
   }
   if (isGroupTask(outcome.node)) view.isGroup = true
   else if (ranNoCache(outcome)) view.noCache = true
+  if (outcome.restoreFailed === true) view.restoreFailed = true
+  const step = outputsFailedOf(outcome)
+  if (step !== undefined) markOutputsFailed(view, step)
   if (outcome.storedDurationMs !== undefined) view.storedDurationMs = outcome.storedDurationMs
   if (outcome.storedCpuMs !== undefined) view.storedCpuMs = outcome.storedCpuMs
   if (outcome.storedPeakRssBytes !== undefined) view.storedPeakRssBytes = outcome.storedPeakRssBytes

package/src/report/forecast.ts

@@ -33,6 +33,11 @@ export interface BlockedTask {
   by: string
 }
 
+/**
+ * One pass over what is unfinished. Called again with the same `finished`
+ * set, that set has only grown: the pass reads only what the last one left
+ * unfinished. Another set starts over.
+ */
 export type Forecast = (
   finished: ReadonlySet<string>,
   running: ReadonlyMap<string, number>,
@@ -78,68 +83,129 @@ export function createForecast(
     }
   }
   for (const n of nodes.values()) place(n)
-  const cost = new Map<string, number>()
-  for (const n of order) {
-    const p = isGroupTask(n) ? undefined : p50s.get(n.id)
-    if (p !== undefined) cost.set(n.id, p)
+  // Each node by its place in `order`, its facts in arrays. A pass built
+  // Maps of the whole graph and read every finished task again, and each
+  // task event forces a frame, each frame a pass: a 20,000-task run spent
+  // ~1.6 s of its main thread on them on a terminal (2026-11-07).
+  const size = order.length
+  const position = new Map(order.map((n, i) => [n.id, i]))
+  const deps = order.map((n) => {
+    const at: number[] = []
+    // A dependency outside the graph never has an end: leaving it out changes nothing.
+    for (const d of n.deps) {
+      const i = position.get(d)
+      if (i !== undefined) at.push(i)
+    }
+    return at
+  })
+  const ids = order.map((n) => n.id)
+  const group = Uint8Array.from(order, (n) => (isGroupTask(n) ? 1 : 0))
+  // NaN: no p50.
+  const cost = new Float64Array(size)
+  const ranks = new Float64Array(size)
+  for (let i = 0; i < size; i++) {
+    const n = order[i]!
+    cost[i] = (group[i] ? undefined : p50s.get(n.id)) ?? NaN
+    ranks[i] = rank.get(n.id) ?? 0
   }
   const workers = Math.max(1, concurrency)
   // The scheduler's own baseline ranking (`rank`), most dependents first,
   // ties in graph order.
-  const position = new Map(order.map((n, i) => [n.id, i]))
-  const before = (a: string, b: string): boolean => {
-    const d = (rank.get(a) ?? 0) - (rank.get(b) ?? 0)
-    return d !== 0 ? d > 0 : position.get(a)! < position.get(b)!
+  const before = (a: number, b: number): boolean => {
+    const d = ranks[a]! - ranks[b]!
+    return d !== 0 ? d > 0 : a < b
   }
+  // Reused by every pass. `end` NaN: no end yet (finished, or not reached);
+  // `since` NaN: not running; `head` -1: no running task its chain waits on.
+  const end = new Float64Array(size)
+  const since = new Float64Array(size)
+  const head = new Int32Array(size)
+  // What the last pass left unfinished, in order: a pass reads only these,
+  // not every finished task again. Rebuilt when `finished` is another set
+  // or shrank; the logger's only grows.
+  const live = new Int32Array(size)
+  let liveCount = -1
+  let lastFinished: ReadonlySet<string> | undefined
+  let lastSize = 0
 
   return (finished, running, nowMs) => {
+    if (finished !== lastFinished || finished.size < lastSize || liveCount < 0) {
+      for (let i = 0; i < size; i++) live[i] = i
+      liveCount = size
+    }
+    let kept = 0
+    for (let r = 0; r < liveCount; r++) {
+      const i = live[r]!
+      if (!finished.has(ids[i]!)) live[kept++] = i
+    }
+    liveCount = kept
+    lastFinished = finished
+    lastSize = finished.size
+    end.fill(NaN)
+    since.fill(NaN)
+    head.fill(-1)
+    for (const [id, at] of running) {
+      const i = position.get(id)
+      if (i !== undefined) since[i] = at
+    }
     let waiting = 0
-    const end = new Map<string, number>()
-    // The running task each node's longest chain waits on.
-    const head = new Map<string, string>()
     let chain = 0
-    let chainEnd: string | undefined
+    let chainEnd = -1
     let work = 0
     let known = false
     // The best `workers` blocked tasks, kept sorted as the pass meets them.
     const held: BlockedTask[] = []
+    const heldAt: number[] = []
     const blockedBy = new Map<string, number>()
-    for (const n of order) {
-      if (finished.has(n.id)) continue
+    for (let r = 0; r < liveCount; r++) {
+      const i = live[r]!
+      const ds = deps[i]!
       let start = 0
-      let via: string | undefined
-      for (const d of n.deps) {
-        const e = end.get(d)
-        if (e !== undefined && (via === undefined || e > start)) {
+      let via = -1
+      for (const d of ds) {
+        const e = end[d]!
+        if (e === e && (via < 0 || e > start)) {
           start = e
           via = d
         }
       }
-      const since = running.get(n.id)
-      if (since !== undefined) head.set(n.id, n.id)
-      else if (via !== undefined && head.has(via)) head.set(n.id, head.get(via)!)
-      const by =
-        since === undefined ? (head.get(n.id) ?? n.deps.find((d) => end.has(d))) : undefined
-      if (by !== undefined && !isGroupTask(n)) {
-        waiting++
-        let at = held.length
-        while (at > 0 && before(n.id, held[at - 1]!.id)) at--
-        if (at < workers) {
-          held.splice(at, 0, { id: n.id, by })
-          if (held.length > workers) held.pop()
+      const s = since[i]!
+      const runs = s === s
+      if (runs) head[i] = i
+      else if (via >= 0 && head[via]! >= 0) head[i] = head[via]!
+      if (!runs && !group[i]) {
+        let by = head[i]!
+        if (by < 0)
+          for (const d of ds)
+            if (end[d] === end[d]) {
+              by = d
+              break
+            }
+        if (by >= 0) {
+          waiting++
+          let at = held.length
+          while (at > 0 && before(i, heldAt[at - 1]!)) at--
+          if (at < workers) {
+            held.splice(at, 0, { id: ids[i]!, by: ids[by]! })
+            heldAt.splice(at, 0, i)
+            if (held.length > workers) {
+              held.pop()
+              heldAt.pop()
+            }
+          }
         }
       }
-      const p = cost.get(n.id)
+      const p = cost[i]!
       let own = 0
-      if (p !== undefined) {
+      if (p === p) {
         known = true
-        own = since === undefined ? p : Math.max(0, p - (nowMs - since))
+        own = runs ? Math.max(0, p - (nowMs - s)) : p
       }
       const e = start + own
-      end.set(n.id, e)
-      if (e > chain || chainEnd === undefined) {
+      end[i] = e
+      if (e > chain || chainEnd < 0) {
         chain = e
-        chainEnd = n.id
+        chainEnd = i
       }
       work += own
     }
@@ -148,7 +214,7 @@ export function createForecast(
       if (d !== undefined && d > 0) blockedBy.set(id, d)
     }
     if (!known) return { waiting, blocked: held, blockedBy }
-    const critical = chainEnd === undefined ? undefined : head.get(chainEnd)
+    const critical = chainEnd < 0 || head[chainEnd]! < 0 ? undefined : ids[head[chainEnd]!]!
     return {
       etaMs: Math.max(chain, work / workers),
       ...(critical !== undefined ? { critical } : {}),