@vzn/vx 0.0.652 → 0.0.653

package/src/orchestrator/task-hash.ts

@@ -1,6 +1,6 @@
 import { lstatSync } from 'node:fs'
 import path from 'node:path'
-import type { TaskConfig } from '../config.js'
+import type { InputStability, TaskConfig } from '../config.js'
 import type { FoldMemo, ProjectFilesCache, WorkspaceFilesCache } from '../cache/index.js'
 import {
   ABSENT_INPUT,
@@ -9,19 +9,27 @@ import {
   behindLink,
   filesToken,
   foldHead,
+  hashFileSeen,
   type InputListing,
   type CacheKeyInput,
   type CacheLayer,
   FILE_HASH_RACY_MS,
   racyWindowMs,
   resolveInputs,
+  stampGrainMs,
   type ResolveInputsArgs,
   type GitFilesCache,
   type TaskInputComponent,
   WORKSPACE_OUTPUT_PREFIX,
 } from '../cache/index.js'
 import type { InputFile, TaskInputs } from '../exec/index.js'
-import { isGroupTask, type TaskNode, type TaskOutcome } from '../graph/index.js'
+import {
+  isGroupTask,
+  KeyFailed,
+  standInsOf,
+  type TaskNode,
+  type TaskOutcome,
+} from '../graph/index.js'
 import { GROUP_VIA } from '../history/index.js'
 import {
   isUserError,
@@ -32,6 +40,7 @@ import {
   UserError,
   xxh3hex,
 } from '../util/index.js'
+import { outputsLeft } from './run-report.js'
 import { expandGroupUpstream, filterUpstreamHashes } from './upstream.js'
 
 /**
@@ -63,8 +72,41 @@ export interface HashCache {
   graph?: string | undefined
 }
 
-export function createHashCache(folds?: FoldMemo): HashCache {
-  return {
+/**
+ * `.vx/workspace`'s `outputs.clean`, after the config stage, by the run's
+ * hash cache: a task without its own runs under it, so its key folds it
+ * (`hashableConfig`). Beside `HashCache`, not on it: that type is the
+ * package's surface.
+ */
+const workspaceClean = new WeakMap<HashCache, boolean>()
+
+/** The workspace's `outputs.clean` the run's keys were derived under. */
+export function workspaceCleanOf(hashCache: HashCache): boolean | undefined {
+  return workspaceClean.get(hashCache)
+}
+
+/**
+ * The `inputs.stability.strategy` of `VX_INPUTS_STABILITY_STRATEGY` (`run`,
+ * beating a task's own) and of `.vx/workspace` (beaten by it), by the run's
+ * hash cache: it decides which saves are kept, so a key folds the one a
+ * task runs under (`hashableConfig`). Beside `HashCache` as the clean is.
+ */
+const inputStrategies = new WeakMap<HashCache, InputStrategies>()
+
+type InputStrategy = InputStability['strategy']
+type InputStrategies = { run?: InputStrategy; workspace?: InputStrategy }
+
+/** The run-level `inputs.stability.strategy` values the run's keys were derived under. */
+export function inputStrategiesOf(hashCache: HashCache): InputStrategies | undefined {
+  return inputStrategies.get(hashCache)
+}
+
+export function createHashCache(
+  folds?: FoldMemo,
+  cleanOutputs?: boolean,
+  strategies?: InputStrategies,
+): HashCache {
+  const hashCache: HashCache = {
     taskConfig: new WeakMap(),
     runtime: new Map(),
     workspaceRuntime: new Map(),
@@ -72,6 +114,10 @@ export function createHashCache(folds?: FoldMemo): HashCache {
     projectFiles: new Map(),
     folds,
   }
+  if (cleanOutputs !== undefined) workspaceClean.set(hashCache, cleanOutputs)
+  if (strategies?.run !== undefined || strategies?.workspace !== undefined)
+    inputStrategies.set(hashCache, strategies)
+  return hashCache
 }
 
 export interface ComputeHashArgs {
@@ -99,6 +145,15 @@ export interface ComputeHashArgs {
   probesAfterWrites?: ReadonlySet<string>
 }
 
+/** A key derivation whose throw is the task's `KeyFailed`: its command never ran. */
+export async function keyOf(derived: Promise<string>): Promise<string> {
+  try {
+    return await derived
+  } catch (err) {
+    throw new KeyFailed(err)
+  }
+}
+
 /**
  * Resolve every input the cache key depends on (file content, env
  * values, upstream hashes, config bytes, forwardArgs) and ask the cache layer to combine them. Used by both
@@ -174,6 +229,12 @@ export interface InputFact {
   path: string
   digest: string
   since: number
+  /**
+   * The ctime of the stat its digest came from, for a file hashed at the
+   * describe (an index OID was not stat'ed): a ctime still equal to it is
+   * not a write since, though the command's start shares its coarse tick.
+   */
+  ctimeMs?: number
   /**
    * For a link, the content behind it (`behindLink`), which the key folds
    * after the link's own target string. An edit there leaves the link
@@ -196,14 +257,21 @@ export async function describeTaskInputs(args: ComputeHashArgs): Promise<{
   inputs: TaskInputs
   facts: InputFact[]
   describedAt: number
+  /** Lists `inputs.upstream` with a non-cacheable upstream's outputs on disk; before any read of it. */
+  readUpstream: () => Promise<void>
   /** An input file the key did not fold that exists now (see `addedInput`). */
   added: () => string | undefined
 }> {
   const hashedAt = Date.now()
   const { input, listings } = await resolveKeyInput(args)
   const sorted = [...input.inputFiles].sort()
+  const ctimes = new Map<string, number>()
   const digests = await Promise.all(
-    sorted.map((f) => input.fileHashes?.get(f) ?? args.cache.hashFile(f).catch(absentOr)),
+    sorted.map(
+      (f) =>
+        input.fileHashes?.get(f) ??
+        hashFileSeen(args.cache, f, (c) => ctimes.set(f, c)).catch(absentOr),
+    ),
   )
   // Learned BEFORE the key folds it: an edit in between then reads as a
   // move at the re-check (a withheld save), never as content the key
@@ -229,6 +297,8 @@ export async function describeTaskInputs(args: ComputeHashArgs): Promise<{
       digest: digests[i]!,
       since: input.fileHashes?.has(f) === true ? indexedAt : hashedAt,
     }
+    const ctimeMs = ctimes.get(f)
+    if (ctimeMs !== undefined) fact.ctimeMs = ctimeMs
     const linked = behind.get(f)
     if (linked !== undefined) fact.behind = linked
     return fact
@@ -241,22 +311,44 @@ export async function describeTaskInputs(args: ComputeHashArgs): Promise<{
     a.hash < b.hash ? -1 : a.hash > b.hash ? 1 : 0,
   )
   // Output lists come from the local index — one SELECT for every upstream
-  // entry. An upstream with no entry (non-cacheable) contributes an empty list.
+  // entry. A non-cacheable upstream keeps no entry: its list is its
+  // declared outputs as they are on disk now (`readUpstream`), as it ran
+  // here and a local dependant reads them here. Left empty, a remote
+  // executor shipped the dependant without them and its result was saved
+  // under the key.
   // Read on first access: the local floor never reads it, and its every
   // miss paid a query over the whole closure (execute-task reads it up
   // front for a plugin executor).
-  const listUpstream = (): TaskInputs['upstream'] => {
+  const listUpstream = (onDisk?: ReadonlyMap<string, string[]>): TaskInputs['upstream'] => {
     const rows = args.cache.loadOutputFilesBatch(graft.map((g) => g.hash))
     return graft.map((g) => {
-      const outputs = (rows.get(g.hash) ?? []).map((r) =>
-        r.path.startsWith(WORKSPACE_OUTPUT_PREFIX)
-          ? r.path.slice(WORKSPACE_OUTPUT_PREFIX.length)
-          : relPosix(input.workspaceRoot, path.join(g.projectDir, r.path)),
-      )
+      const outputs =
+        onDisk?.get(g.taskId) ??
+        (rows.get(g.hash) ?? []).map((r) =>
+          r.path.startsWith(WORKSPACE_OUTPUT_PREFIX)
+            ? r.path.slice(WORKSPACE_OUTPUT_PREFIX.length)
+            : relPosix(input.workspaceRoot, path.join(g.projectDir, r.path)),
+        )
       return { taskId: g.taskId, hash: g.hash, outputs }
     })
   }
   let upstream: TaskInputs['upstream'] | undefined
+  const readUpstream = async (): Promise<void> => {
+    if (upstream !== undefined) return
+    const uncached = expandGroupUpstream(args.upstream).filter(
+      (u) => u.hash !== undefined && u.node.config.cache !== true,
+    )
+    // No nested project is cut out: each file found is on this disk, where
+    // a local run of the dependant reads it too.
+    const onDisk = new Map(
+      await Promise.all(
+        uncached.map(
+          async (u) => [u.node.id, await outputsLeft(u.node, input.workspaceRoot, [])] as const,
+        ),
+      ),
+    )
+    upstream ??= listUpstream(onDisk)
+  }
   // A listing from the run's enumeration is as old as it; one listed with
   // no run cache was listed during this describe.
   const listedAt = args.gitFilesCache === undefined ? hashedAt : indexedAt
@@ -264,6 +356,7 @@ export async function describeTaskInputs(args: ComputeHashArgs): Promise<{
     hash,
     facts,
     describedAt: hashedAt,
+    readUpstream,
     added: () => addedInput(listings, new Set(input.inputFiles), listedAt),
     inputs: {
       files,
@@ -287,11 +380,15 @@ export async function describeTaskInputs(args: ComputeHashArgs): Promise<{
  * a file whose ctime is older than its fact (by the racy window, as git
  * judges its index) has not been written since, and one that is not gets
  * hashed again and compared. ctime, because no writer can set it back. A
- * file that is gone has moved. One written at or after `commandFrom` (the
- * describe just before the command) has moved whatever it holds now: an
- * input changed and changed BACK while the command ran matched its digest
- * again, and the entry saved the other content's output under the key (item
- * 1015). A write before it — an upstream's — is judged by content, as before.
+ * file that is gone has moved. One written after `commandFrom` (read off
+ * the file system's clock, `fsClockNow`, just after the describe) has moved
+ * whatever it holds now: an input changed and changed BACK while the
+ * command ran matched its digest again, and the entry saved the other
+ * content's output under the key (item 1015). A write before it — an
+ * upstream's — is judged by content, as before. A coarse clock dates a
+ * write in `commandFrom`'s own tick at it, before or after: such a ctime is
+ * the upstream's when the describe's stat saw it already
+ * (`InputFact.ctimeMs`), else the command's.
  * `byBytes` (`inputs.stability.strategy: 'mutable'`) judges a write during
  * the command by content too, when it answers true for that file and its
  * ctime: the task says the same bytes are its own identical rewrite
@@ -318,9 +415,14 @@ export async function movedInput(
     // Back since the key folded it gone.
     if (f.digest === ABSENT_INPUT) return f.path
     // A whole-second stamp may stand for any write in its second (two for
-    // an even one, FAT's tick), one during the command included
-    // (`racyWindowMs`).
-    if (commandFrom !== undefined && ctimeMs >= commandFrom - racyWindowMs(ctimeMs, 0)) {
+    // an even one, FAT's tick), one during the command included. No lag
+    // widens it: `commandFrom` is on the file system's clock already.
+    const grainMs = stampGrainMs(ctimeMs)
+    if (
+      commandFrom !== undefined &&
+      (ctimeMs > commandFrom ||
+        (ctimeMs >= commandFrom - grainMs && (grainMs > 0 || f.ctimeMs !== ctimeMs)))
+    ) {
       if (byBytes?.(f.path, ctimeMs) !== true) return f.path
       suspects.push(f)
       continue
@@ -340,14 +442,27 @@ export async function movedInput(
   return undefined
 }
 
-/** A task's `filesToken`, as its key's fold memo probe takes it. */
+/**
+ * A task's `filesToken`, as its key's fold memo probe takes it: the fields
+ * of `inputsArgs` the token reads, and no more. Built through `inputsArgs`,
+ * two spread objects per call were ~3 ms of 3,202 (2026-11-07).
+ */
 export function taskFilesToken(
-  args: Pick<
-    ComputeHashArgs,
-    'node' | 'workspaceRoot' | 'nestedProjectDirs' | 'gitFilesCache' | 'hashCache'
-  >,
+  args: Pick<ComputeHashArgs, 'node' | 'workspaceRoot' | 'nestedProjectDirs'> & {
+    gitFilesCache: GitFilesCache
+  },
 ): string | undefined {
-  return filesToken(inputsArgs({ ...args, upstream: [], cache: undefined as never }))
+  const node = args.node
+  const out = node.config.outputs
+  return filesToken({
+    projectDir: node.projectDir,
+    workspaceRoot: args.workspaceRoot,
+    inputs: node.config.inputs,
+    ownOutputs: out?.files ?? [],
+    ownWorkspaceOutputs: out?.workspaceFiles ?? [],
+    nestedProjectDirs: args.nestedProjectDirs,
+    gitFilesCache: args.gitFilesCache,
+  })
 }
 
 /** What `resolveInputs` is asked for a task's key. */
@@ -397,6 +512,7 @@ async function resolveKeyInput(
     cfg.inputs?.tasks,
     args.node.projectName,
     args.node.id,
+    standInsOf(args.node),
   )
   const upstreamHashes = upstreamPairs.map(([, hash]) => hash)
   // hash → upstream task id, for capture-row naming only (not folded).
@@ -487,7 +603,7 @@ export function hashTaskConfig(cfg: TaskConfig, hashCache?: HashCache): string {
   if (hashCache) {
     const cached = hashCache.taskConfig.get(cfg)
     if (cached !== undefined) return cached
-    const hash = xxh3hex(JSON.stringify(hashableConfig(cfg)))
+    const hash = xxh3hex(JSON.stringify(hashableConfig(asRun(cfg, hashCache))))
     hashCache.taskConfig.set(cfg, hash)
     return hash
   }
@@ -495,7 +611,36 @@ export function hashTaskConfig(cfg: TaskConfig, hashCache?: HashCache): string {
 }
 
 /**
- * Project the config for hashing: `exec.remote` is a pure PLACEMENT hint
+ * The config as the task runs it, where a run-level value changes what is
+ * saved: folded as the task's own field would be, and only then, so every
+ * key that never ran under one stays.
+ *
+ * A workspace `outputs.clean: false` saves whatever the outputs held before
+ * the command, a leftover included: a key that missed it hit that entry
+ * once the clean was back on. An `inputs.stability.strategy` other than
+ * `'immutable'` keeps a save the check withholds (`'unchecked'`, one whose
+ * input moved under the command): unfolded, that entry was hit once the
+ * check was back on, and a formatter it covered never ran.
+ */
+function asRun(cfg: TaskConfig, hashCache: HashCache): TaskConfig {
+  let out = cfg
+  if (workspaceClean.get(hashCache) === false && cfg.outputs && cfg.outputs.clean === undefined) {
+    out = { ...out, outputs: { ...cfg.outputs, clean: false } }
+  }
+  const strategies = inputStrategies.get(hashCache)
+  const inputs = cfg.inputs
+  if (strategies !== undefined && inputs !== undefined && cfg.exec !== undefined) {
+    const own = inputs.stability?.strategy
+    const runs = strategies.run ?? own ?? strategies.workspace ?? 'immutable'
+    if (runs !== (own ?? 'immutable')) {
+      out = { ...out, inputs: { ...inputs, stability: { ...inputs.stability, strategy: runs } } }
+    }
+  }
+  return out
+}
+
+/**
+ * Project the config for hashing: `exec.remote` (and `exec.worker`) is a pure PLACEMENT hint
  * with zero effect on outputs, so it is stripped — pinning a task to this
  * machine never busts a cache. It must not: the whole contract of a remote
  * executor is that the same command over the same inputs produces the
@@ -521,12 +666,24 @@ function hashableConfig(cfg: TaskConfig): unknown {
   // writes a config the lock never sorted (2026-10-25).
   const env = cfg.inputs?.env
   const pass = cfg.exec?.env?.passThrough
-  if (cfg.exec?.remote === undefined && sortedNames(env) && sortedNames(pass))
+  const secret = cfg.exec?.env?.secret
+  if (
+    cfg.exec?.remote === undefined &&
+    cfg.exec?.worker === undefined &&
+    sortedNames(env) &&
+    sortedNames(pass) &&
+    sortedNames(secret)
+  )
     return keysSorted(cfg)
-  const { remote: _remote, ...execRest } = cfg.exec ?? ({} as NonNullable<TaskConfig['exec']>)
+  const {
+    remote: _remote,
+    worker: _worker,
+    ...execRest
+  } = cfg.exec ?? ({} as NonNullable<TaskConfig['exec']>)
   // Env names are sets: each value is read by its name, so their order
   // keys nothing (conformance env/declared-order-irrelevant).
   if (!sortedNames(pass)) execRest.env = { ...execRest.env, passThrough: [...pass!].sort() }
+  if (!sortedNames(secret)) execRest.env = { ...execRest.env, secret: [...secret!].sort() }
   const out: TaskConfig = cfg.exec === undefined ? { ...cfg } : { ...cfg, exec: execRest }
   if (!sortedNames(env)) out.inputs = { ...cfg.inputs!, env: [...env!].sort() }
   return keysSorted(out)

package/src/orchestrator/telemetry-host.ts

@@ -12,7 +12,7 @@
 // See docs/design/observability-architecture-2026-06.md §1, §3.
 
 import type { EventBus } from '../report/index.js'
-import type { TaskNode } from '../graph/index.js'
+import type { TaskNode, TaskOutcome } from '../graph/index.js'
 import type {
   RunContextRecord,
   RunSummaryRecord,
@@ -31,6 +31,8 @@ export interface TelemetryHandle {
   emitSummary(summary: RunSummaryRecord): void
   /** Await every sink's flush (crash-isolated, time-bounded by the sink), each as a `<plugin>:telemetry` step. */
   flush(step?: <T>(id: string, work: () => Promise<T>) => Promise<T>): Promise<void>
+  /** Send the held `task.end` of each server not in `keep`; see `TelemetrySource.endServers`. */
+  endServers(final: readonly TaskOutcome[], keep?: ReadonlySet<string>): void
   /** Remove the bus subscription. Idempotent. */
   dispose(): void
   /** Sample a running task's process tree; see `TelemetrySource.track`. */
@@ -47,12 +49,17 @@ const RECORD_KINDS: Record<TelemetryRecord['kind'], true> = {
   'run.end': true,
 }
 
-/** Reject anything that is not sink-shaped, naming what arrived. */
+/**
+ * Reject anything that is not sink-shaped, naming what arrived, and return
+ * a copy of its members read once here. The sink is user input: a getter
+ * read again downstream, outside this try, threw past every guard and
+ * failed a green run.
+ */
 function checkSink(sink: unknown): TelemetrySink {
   if (typeof sink !== 'object' || sink === null) {
     throw new Error(`telemetry sink must be an object, got ${sink === null ? 'null' : typeof sink}`)
   }
-  const wants = (sink as TelemetrySink).wants
+  const { name, wants, onRecord, onRunSummary, flush } = sink as TelemetrySink
   if (wants !== undefined && !Array.isArray(wants)) {
     throw new Error(`telemetry sink 'wants' must be an array, got ${typeof wants}`)
   }
@@ -69,13 +76,24 @@ function checkSink(sink: unknown): TelemetrySink {
   // subscribed to the bus and heard nothing — the plugin was "on" and did
   // nothing. Refused here, which the caller turns into the same warning
   // a throwing hook gets: never a failed run.
-  const s = sink as TelemetrySink
-  if (typeof s.onRecord !== 'function' && typeof s.onRunSummary !== 'function') {
+  if (typeof onRecord !== 'function' && typeof onRunSummary !== 'function') {
     throw new Error(
       'telemetry sink handles nothing: neither onRecord nor onRunSummary is a function',
     )
   }
-  return s
+  for (const [key, fn] of Object.entries({ onRecord, onRunSummary, flush })) {
+    if (fn !== undefined && typeof fn !== 'function') {
+      throw new Error(`telemetry sink '${key}' must be a function, got ${typeof fn}`)
+    }
+  }
+  // Bound: a sink written as a class reads its own state through `this`.
+  return {
+    ...(typeof name === 'string' ? { name } : {}),
+    ...(wants !== undefined ? { wants: [...wants] } : {}),
+    ...(onRecord !== undefined ? { onRecord: onRecord.bind(sink) } : {}),
+    ...(onRunSummary !== undefined ? { onRunSummary: onRunSummary.bind(sink) } : {}),
+    ...(flush !== undefined ? { flush: flush.bind(sink) } : {}),
+  }
 }
 
 /**
@@ -147,6 +165,7 @@ export async function subscribeTelemetry(
   return {
     emitSummary: (summary) => source.emitSummary(summary),
     flush: (step) => source.flush(step),
+    endServers: (final, keep) => source.endServers(final, keep),
     dispose,
     ...(source.track !== undefined ? { track: source.track } : {}),
   }

package/src/orchestrator/telemetry.ts

@@ -10,7 +10,9 @@
 // rendering-oriented `WireEvent` stream.
 //
 // A sink is observe-only BY CONSTRUCTION: its only input is a plain-data
-// record (one object every sink receives, not frozen); its `TelemetryContext` carries read-only metadata and NO mutable
+// record (one object every sink receives, frozen once, deep: a sink that
+// cleared a `task.end`'s `failedAttempts` cleared the outcome's own and
+// crashed the run's failure log); its `TelemetryContext` carries read-only metadata and NO mutable
 // run handle (no bus, no Cache, no RunRequest). There is no API path from a
 // sink back into scheduling/caching/exec — telemetry provably cannot change
 // what or how tasks run. Contrast `cache`/`executor`, which return objects
@@ -34,7 +36,8 @@ import {
   teardownTimeoutMs,
   thrownMessage,
 } from '../util/index.js'
-import type { RunEvent, RunEventSubscriber } from '../report/index.js'
+import { neverStarted, type RunEvent, type RunEventSubscriber } from '../report/index.js'
+import { freezeDeep } from './read-only.js'
 
 /** Bumped when the record shape changes. Readers MUST check `v`. */
 export const TELEMETRY_SCHEMA_VERSION = 4
@@ -256,14 +259,20 @@ export interface RunSummaryRecord {
   startedAt: number
   endedAt: number
   totalDurationMs: number
+  /** Every task the run had, as the terminal's total: `tasks` plus `notRunCount`. */
   taskCount: number
   failedCount: number
   /**
-   * Tasks a shutdown signal or an embedder's abort killed (v3). They are
-   * not in `tasks`, which holds real runs only, so without this a stopped
-   * run read as a failure with nothing failed (item 851).
+   * Tasks a shutdown signal or an embedder's abort killed once started (v3),
+   * each in `tasks` as `aborted`: without the count a stopped run read as a
+   * failure with nothing failed (item 851).
    */
   abortedCount: number
+  /**
+   * Tasks the stop reached before they started: the terminal's `not run`.
+   * Not in `tasks`, since they never ran. Absent when none. Additive.
+   */
+  notRunCount?: number
   hitCount: number
   /** Hits by the layer that answered, up-to-date ones included. */
   hitLocalCount: number
@@ -318,8 +327,9 @@ export interface RunStage {
  * computed: both a local `run()` and the distributed controller build their
  * `TaskTelemetry[]` (each via `deriveCacheSource`) and call this, so a
  * distributed run and a local run produce byte-identical summaries and land in
- * the same ingest. The per-task tallies (taskCount / failedCount /
- * hitLocal|Remote|Count) derive from `tasks`; `totalDurationMs` (wall time) and
+ * the same ingest. The per-task tallies (failedCount /
+ * hitLocal|Remote|Count) derive from `tasks`, and `taskCount` from `tasks`
+ * plus `notRunCount`; `totalDurationMs` (wall time) and
  * `exitOk` (the run's overall verdict — which counts skipped tasks beyond the
  * recorded task list) are run-level facts and are passed in.
  */
@@ -332,6 +342,7 @@ export function assembleRunSummary(
     totalDurationMs: number
     exitOk: boolean
     abortedCount: number
+    notRunCount?: number
     stages?: readonly RunStage[]
     uploads?: { count: number; bytes: number; ms: number; failed: number }
     machines?: readonly MachineReport[]
@@ -360,9 +371,14 @@ export function assembleRunSummary(
     startedAt: timing.startedAt,
     endedAt: timing.endedAt,
     totalDurationMs: timing.totalDurationMs,
-    taskCount: tasks.length,
+    // The terminal's total: the tasks a stop reached first are not in
+    // `tasks`, yet a summary of 2 beside a terminal's 3 read as a lost task.
+    taskCount: tasks.length + (timing.notRunCount ?? 0),
     failedCount,
     abortedCount: timing.abortedCount,
+    ...(timing.notRunCount !== undefined && timing.notRunCount > 0
+      ? { notRunCount: timing.notRunCount }
+      : {}),
     hitCount: hitLocalCount + hitRemoteCount,
     hitLocalCount,
     hitRemoteCount,
@@ -383,7 +399,11 @@ export function assembleRunSummary(
   }
 }
 
-/** A telemetry consumer. Observe-only: receives records, holds no run handle. */
+/**
+ * A telemetry consumer. Observe-only: receives records, frozen, holds no
+ * run handle. A write to a record throws, and the sink is switched off for
+ * the run as for any throw.
+ */
 export interface TelemetrySink {
   readonly name?: string
   /**
@@ -395,7 +415,11 @@ export interface TelemetrySink {
   readonly wants?: ReadonlyArray<TelemetryRecord['kind']>
   /** A streaming record. MUST return promptly — buffer; do NOT await I/O. */
   onRecord?(record: TelemetryRecord): void
-  /** The per-run summary, at run:end. MUST return promptly. */
+  /**
+   * The per-run summary, once at run:end; a requested persistent task the
+   * foreground keeps sends it when that task exits, so it reads the run's
+   * final verdict. MUST return promptly.
+   */
   onRunSummary?(summary: RunSummaryRecord): void
   /**
    * Drain buffered data. Awaited at run:end under a deadline — a sink that
@@ -433,6 +457,12 @@ export interface TelemetrySource {
    * each as the step `<plugin>:telemetry` through `step`.
    */
   flush(step?: <T>(id: string, work: () => Promise<T>) => Promise<T>): Promise<void>
+  /**
+   * Send the held `task.end` of each persistent task not in `keep`, read
+   * from `final` (by id): a server's outcome can change after its ready,
+   * when it dies before the run stops it or ends a kept session.
+   */
+  endServers(final: readonly TaskOutcome[], keep?: ReadonlySet<string>): void
   /**
    * Sample this task's process tree every `SAMPLE_MS` until the returned
    * function is called, the root exits, or the run ends. Present only when
@@ -602,6 +632,9 @@ export function createTelemetrySource(args: {
   }
 
   function deliver(record: TelemetryRecord): void {
+    // Once, for every sink: a `task.end` holds the outcome's own arrays, and
+    // a sink that cleared `failedAttempts` crashed the run's failure log.
+    freezeDeep(record)
     for (const sink of sinks) {
       if (disabled.has(sink) || sink.onRecord === undefined) continue
       const kinds = sink.wants ?? DEFAULT_KINDS
@@ -667,6 +700,16 @@ export function createTelemetrySource(args: {
       case 'task:complete': {
         const { node, outcome } = event
         if (node.config.exec === undefined) return // group task
+        // The stop reached it first: no task.start, and the summary counts
+        // it in `notRunCount`, not `tasks`, so no task.end either.
+        if (neverStarted(outcome)) return
+        // A server's end waits for its final outcome (`endServers`): one
+        // task.end per task, and the one sent at ready read `success` over
+        // a crash after it.
+        if (node.config.exec.persistent !== undefined) {
+          heldEnds.set(node.id, { ts, outcome })
+          return
+        }
         deliver({
           v: TELEMETRY_SCHEMA_VERSION,
           kind: 'task.end',
@@ -687,10 +730,26 @@ export function createTelemetrySource(args: {
     }
   }
 
+  const heldEnds = new Map<string, { ts: number; outcome: TaskOutcome }>()
   return {
     subscriber,
     ...(wantsSample ? { track } : {}),
+    endServers(final, keep) {
+      for (const [id, held] of heldEnds) {
+        if (keep?.has(id) === true) continue
+        heldEnds.delete(id)
+        const outcome = final.find((o) => o.node.id === id) ?? held.outcome
+        deliver({
+          v: TELEMETRY_SCHEMA_VERSION,
+          kind: 'task.end',
+          runId,
+          ts: held.ts,
+          ...taskTelemetryOf(outcome),
+        })
+      }
+    },
     emitSummary(summary: RunSummaryRecord): void {
+      freezeDeep(summary)
       for (const sink of sinks) {
         if (disabled.has(sink) || sink.onRunSummary === undefined) continue
         try {
@@ -712,9 +771,12 @@ export function createTelemetrySource(args: {
       // accessed_at bumps and every later plugin's teardown lost with it.
       // Sinks race concurrently, so each still gets the whole budget.
       const deadline = new AbortController()
+      // The deadline's warning names the sinks still out, as a throw names its own.
+      const pending = new Set<TelemetrySink>()
       const all = Promise.all(
         sinks.map(async (sink) => {
           if (disabled.has(sink) || sink.flush === undefined) return
+          pending.add(sink)
           try {
             await step(`${args.owners?.get(sink) ?? label(sink)}:telemetry`, () =>
               sink.flush!(deadline.signal),
@@ -723,12 +785,17 @@ export function createTelemetrySource(args: {
             // A flush failure can never break the run — but it still costs
             // this sink its export, so say so rather than dropping it.
             warn?.(`[vx] telemetry sink '${label(sink)}' failed to flush: ${thrownMessage(err)}`)
+          } finally {
+            pending.delete(sink)
           }
         }),
       )
       if (!(await settleWithin(all, ms))) {
         deadline.abort(new Error(`telemetry flush deadline (${ms}ms)`))
-        warn?.(`[vx] telemetry flush timed out after ${ms}ms; buffered records lost`)
+        const names = [...pending].map((s) => `'${label(s)}'`).join(', ')
+        warn?.(
+          `[vx] telemetry flush timed out after ${ms}ms (sink${pending.size === 1 ? '' : 's'} ${names}); buffered records lost`,
+        )
         // A sink told of the abort says what it lost as its request ends; the
         // run ended first and that line was lost (vx-otel, on CI). Briefly.
         await settleWithin(all, ABORT_GRACE_MS)
@@ -767,7 +834,8 @@ export function taskTelemetryOf(o: TaskOutcome): TaskTelemetry {
     project: o.node.projectName,
     task: o.node.taskName,
     status: o.status,
-    cacheSource: deriveCacheSource(o.status),
+    // A hit whose restore failed is in no cache column: neither a miss nor a hit.
+    cacheSource: o.restoreFailed === true ? 'none' : deriveCacheSource(o.status),
     exitCode: o.exitCode,
     durationMs: o.durationMs,
   }

package/src/orchestrator/upstream.ts

@@ -1,5 +1,6 @@
 import {
   selectFoldedDeps,
+  type StandIns,
   type FoldCandidate,
   type TaskNode,
   type TaskOutcome,
@@ -72,10 +73,11 @@ export function filterUpstreamHashes(
   filter: readonly string[] | undefined,
   selfProjectName: string,
   selfTaskId: string,
+  standIns?: StandIns,
 ): Array<[upstreamTaskId: string, hash: string]> {
   const candidates: FoldCandidate[] = []
   for (const u of upstream) if (u.hash) candidates.push({ node: u.node, unit: u.hash })
-  return selectFoldedDeps(candidates, filter, selfProjectName, selfTaskId).map((c) => [
+  return selectFoldedDeps(candidates, filter, selfProjectName, selfTaskId, standIns).map((c) => [
     c.node.id,
     c.unit,
   ])