@vzn/vx 0.0.15 → 0.0.18

package/src/orchestrator/task-log-buffer.ts

@@ -0,0 +1,291 @@
+// The shared per-task log capture primitive (task-logs-2026-07 §1). ONE
+// bounded-tail buffer every telemetry sink uses — the cloud client sink (local
+// runs), the cloud serve sink (delegated runs), the dist scheduler, and the
+// OTLP logs exporter — so the capping rules can't drift between capture sites.
+//
+// It lives in CORE, beside the telemetry contract, for the same reason
+// `assembleRunSummary` does: two sinks that each roll their own bounded buffer
+// is how the retention rule forks, and a forked rule means two sinks disagree
+// about which task's output survived. A sink is the only consumer — core
+// itself never captures logs — but the vocabulary it decides in (`TaskStatus`,
+// `CacheSource`, "a hit's bytes belong to the run that executed") is core's.
+//
+// The law: a log-spewing task can never OOM the capturer. Two caps enforce it:
+//   - per task: only the last TASK_LOG_TAIL_CHARS of merged stdout+stderr is
+//     retained (chunks evicted from the head, no concatenation until drain);
+//   - per run: only RUN_LOG_BUDGET_CHARS of RETAINED completed tails ship, and
+//     failed tails are never evicted by successes — a failure's output is the
+//     one thing you always want to read.
+//
+// Both caps report themselves: an evicted or head-trimmed task still ships,
+// carrying how much was dropped, so "we truncated this" never reads as "this
+// task printed nothing".
+
+import type { TaskStatus } from '../graph/index.js'
+import type { CacheSource } from './telemetry.js'
+
+/** Version of the drained-bundle shape below. It is the canonical drained
+ *  logs format, not one transport's: cloud's `POST /v1/ingest/logs` accepts
+ *  it verbatim, and any other sink ships the same object. */
+export const LOG_WIRE_VERSION = 1
+
+/**
+ * Per-task tail retained (UTF-16 units — a cheap capture-side proxy; the store
+ * records true stored bytes). Merged streams: what a terminal actually shows.
+ */
+export const TASK_LOG_TAIL_CHARS = 128 * 1024
+
+/** Total RETAINED completed-task chars a single run ships. */
+export const RUN_LOG_BUDGET_CHARS = 4 * 1024 * 1024
+
+export interface TaskLogEntry {
+  taskId: string
+  /** The task's cache key when known — the hit→executed-run resolution key. */
+  hash?: string
+  status: 'success' | 'failed'
+  /** Merged stdout+stderr in arrival order, tail-capped, ANSI preserved. */
+  content: string
+  /** Chars emitted before capping. */
+  charsFull: number
+  /** Chars dropped from the head; 0 = complete. */
+  truncatedHeadChars: number
+}
+
+export interface TaskLogBundle {
+  v: typeof LOG_WIRE_VERSION
+  runId: string
+  workspaceId: string
+  tasks: TaskLogEntry[]
+}
+
+/** Retained-until-drain state for one completed task. */
+interface Retained {
+  taskId: string
+  hash?: string
+  status: 'success' | 'failed'
+  /** Emptied when the run budget evicts this task; `chars` goes to 0 and the
+   *  dropped count moves to `truncatedHeadChars` (see `evictToBudget`). */
+  chunks: string[]
+  chars: number
+  charsFull: number
+  truncatedHeadChars: number
+  /** Insertion order — the eviction tiebreak (oldest first). */
+  seq: number
+}
+
+/** In-flight (not yet ended) accumulation for one task. */
+interface InFlight {
+  chunks: string[]
+  chars: number
+  charsFull: number
+  truncatedHeadChars: number
+}
+
+/**
+ * Bounded per-run capture. `append` keeps a chunk LIST + running char count per
+ * task, evicting whole chunks from the head past `TASK_LOG_TAIL_CHARS` — no
+ * string concatenation until `drain`, so a cache-hit replay (one big chunk) is
+ * one array push, zero copies. `finish` decides retention; `drain` emits the
+ * bundle, failures first.
+ */
+/**
+ * Per-retained-chunk overhead, in char-equivalents, charged against the RUN
+ * budget on top of the characters themselves.
+ *
+ * The budget exists to bound MEMORY, but a char count is not one: a task that
+ * emits a byte at a time (unbuffered progress output) stores the same content
+ * as thousands of separate strings, each with its own header and allocation.
+ * MEASURED on Bun 1.4: 1 000 000 chars arrive as one chunk for ~1.4 MB of
+ * marginal RSS, and as 1 000 000 one-char chunks for ~30 MB — so a 4 MiB
+ * char budget could hold ~80 MB of actual memory. Charging each chunk ~24
+ * char-equivalents makes the budget track what it is trying to limit; chunky
+ * output is unaffected (one 128 KiB chunk pays 24 on 131 072).
+ *
+ * The per-task TAIL cap deliberately stays a pure char count — it bounds how
+ * much a reader is shown, which is a different question from how much the
+ * process holds.
+ */
+const CHUNK_OVERHEAD_CHARS = 24
+
+/** What one retained entry costs the run budget: content + per-chunk overhead. */
+function budgetCost(chars: number, chunkCount: number): number {
+  return chars + chunkCount * CHUNK_OVERHEAD_CHARS
+}
+
+export class TaskLogBuffer {
+  private readonly inFlight = new Map<string, InFlight>()
+  private readonly retained = new Map<string, Retained>()
+  private retainedChars = 0
+  private seq = 0
+
+  append(taskId: string, chunk: string): void {
+    if (chunk.length === 0) return
+    let acc = this.inFlight.get(taskId)
+    if (acc === undefined) {
+      acc = { chunks: [], chars: 0, charsFull: 0, truncatedHeadChars: 0 }
+      this.inFlight.set(taskId, acc)
+    }
+    acc.chunks.push(chunk)
+    acc.chars += chunk.length
+    acc.charsFull += chunk.length
+    // Evict whole chunks from the head until under the per-task tail cap. A
+    // single over-cap chunk is sliced to its tail (the only place we copy).
+    while (acc.chars > TASK_LOG_TAIL_CHARS && acc.chunks.length > 1) {
+      const dropped = acc.chunks.shift()!
+      acc.chars -= dropped.length
+      acc.truncatedHeadChars += dropped.length
+    }
+    if (acc.chars > TASK_LOG_TAIL_CHARS && acc.chunks.length === 1) {
+      const only = acc.chunks[0]!
+      const keep = only.slice(only.length - TASK_LOG_TAIL_CHARS)
+      acc.truncatedHeadChars += only.length - keep.length
+      acc.chunks[0] = keep
+      acc.chars = keep.length
+    }
+  }
+
+  /**
+   * task.end: decide retention.
+   *   - cacheSource !== 'miss' (a hit) → DROP: the executed run already stored
+   *     these bytes; a hit resolves by hash to that run.
+   *   - 'skipped' / 'aborted' → DROP (no meaningful output; a tearing-down run).
+   *   - a success/failed MISS → RETAIN the tail, under the run budget: adding a
+   *     task that would exceed `RUN_LOG_BUDGET_CHARS` evicts oldest retained
+   *     SUCCESS tails first; failed tails are evicted only by newer failed tails
+   *     (oldest first) once failures alone exceed the budget.
+   */
+  finish(taskId: string, status: TaskStatus, cacheSource: CacheSource, hash?: string): void {
+    const acc = this.inFlight.get(taskId)
+    this.inFlight.delete(taskId)
+    if (cacheSource !== 'miss') return
+    if (status !== 'success' && status !== 'failed') return
+    if (acc === undefined) return
+
+    const entry: Retained = {
+      taskId,
+      ...(hash !== undefined ? { hash } : {}),
+      status,
+      chunks: acc.chunks,
+      chars: acc.chars,
+      charsFull: acc.charsFull,
+      truncatedHeadChars: acc.truncatedHeadChars,
+      seq: this.seq++,
+    }
+    // Replace an earlier retention for the same task id (a retried task ends
+    // once per attempt only via the winning outcome, but be defensive).
+    const prior = this.retained.get(taskId)
+    if (prior !== undefined) this.retainedChars -= budgetCost(prior.chars, prior.chunks.length)
+    this.retained.set(taskId, entry)
+    this.retainedChars += budgetCost(entry.chars, entry.chunks.length)
+    this.evictToBudget()
+  }
+
+  /**
+   * Take ONE finished task's retained tail as a wire entry, removing it from
+   * the buffer (so a later `drain` won't re-ship it). Used by per-task
+   * incremental delivery: `finish()` retains, then the sink `takeEntry()`s and
+   * ships that one task. Returns undefined for a task that wasn't retained (a
+   * cache hit / skipped / still in-flight / unknown id) — the task result still
+   * ships; there is just no log tail to send.
+   */
+  takeEntry(taskId: string): TaskLogEntry | undefined {
+    const e = this.retained.get(taskId)
+    if (e === undefined) return undefined
+    this.retained.delete(taskId)
+    this.retainedChars -= budgetCost(e.chars, e.chunks.length)
+    return {
+      taskId: e.taskId,
+      ...(e.hash !== undefined ? { hash: e.hash } : {}),
+      status: e.status,
+      content: e.chunks.join(''),
+      charsFull: e.charsFull,
+      truncatedHeadChars: e.truncatedHeadChars,
+    }
+  }
+
+  /** Everything retained, failures first, ready to ship/store. */
+  drain(runId: string, workspaceId: string): TaskLogBundle {
+    const entries = [...this.retained.values()].sort((a, b) => {
+      // Failures ahead of successes; within a tier, oldest first (stable).
+      const fa = a.status === 'failed' ? 0 : 1
+      const fb = b.status === 'failed' ? 0 : 1
+      return fa !== fb ? fa - fb : a.seq - b.seq
+    })
+    return {
+      v: LOG_WIRE_VERSION,
+      runId,
+      workspaceId,
+      tasks: entries.map((e) => ({
+        taskId: e.taskId,
+        ...(e.hash !== undefined ? { hash: e.hash } : {}),
+        status: e.status,
+        content: e.chunks.join(''),
+        charsFull: e.charsFull,
+        truncatedHeadChars: e.truncatedHeadChars,
+      })),
+    }
+  }
+
+  /** Retained-tasks count (test/observability). */
+  size(): number {
+    return this.retained.size
+  }
+
+  /**
+   * Budget charged right now, in char-equivalents (test/observability).
+   *
+   * Exposed because the charge/release symmetry is the invariant that keeps
+   * the cap meaningful and the one that fails SILENTLY: a release that does
+   * not match its charge drifts `retainedChars` a little per task, and a
+   * long run ends up bounding something other than what it thinks.
+   */
+  budgetUsed(): number {
+    return this.retainedChars
+  }
+
+  /**
+   * Evict until `retainedChars <= RUN_LOG_BUDGET_CHARS`. Successes go first
+   * (oldest by seq); only when failures ALONE still exceed the budget do the
+   * oldest failures go — a failure is never dropped to keep a success.
+   *
+   * An evicted task is degraded to a STUB (no content, `truncatedHeadChars`
+   * = everything it emitted) rather than removed. Dropping the entry makes a
+   * budget eviction indistinguishable from a task that printed nothing — the
+   * reader gets "no logs captured" either way, which is a confident false
+   * statement in the one case where they most need the output (a run with
+   * enough failing tasks to blow the budget). A stub costs a few dozen bytes,
+   * keeps the `content.length === charsFull - truncatedHeadChars` accounting
+   * honest, and renders through the truncation banner the per-task cap
+   * already uses. It is also what the STORE does when its own run budget runs
+   * out (`Analytics.ingestLogs` slices to empty and adds the remainder to
+   * `truncated_head`) — the two sides now degrade the same way.
+   *
+   * The cost is a stub per evicted task in the end-of-run bundle: measured at
+   * ~190 JSON bytes each, so ~1.9 MiB at the 10k-task scale target and ~9.3 MiB
+   * at 50k, against the 16 MiB `/v1/ingest/logs` cap. A run evicting past ~66k
+   * tasks would 413 the batch — remote (6x the scale target), and it does not
+   * touch the connected default, which ships per task as it finishes.
+   */
+  private evictToBudget(): void {
+    if (this.retainedChars <= RUN_LOG_BUDGET_CHARS) return
+    const order = [...this.retained.values()].sort((a, b) => {
+      const sa = a.status === 'success' ? 0 : 1
+      const sb = b.status === 'success' ? 0 : 1
+      if (sa !== sb) return sa - sb // successes are dropped first, always
+      // Within a tier the tiebreak DIFFERS by status, and the failed one is
+      // the point: when failures alone blow the budget, the FIRST failure is
+      // usually the root cause and the later ones its cascade, so it must be
+      // the last thing stubbed. Successes keep oldest-first — none of them
+      // is more interesting than another, and recency is the better guess.
+      return a.status === 'failed' ? b.seq - a.seq : a.seq - b.seq
+    })
+    for (const e of order) {
+      if (this.retainedChars <= RUN_LOG_BUDGET_CHARS) break
+      if (e.chars === 0) continue // already a stub — frees nothing
+      this.retainedChars -= budgetCost(e.chars, e.chunks.length)
+      e.truncatedHeadChars += e.chars
+      e.chunks = []
+      e.chars = 0
+    }
+  }
+}

package/src/orchestrator/telemetry-host.ts

@@ -31,6 +31,18 @@ export interface TelemetryHandle {
   dispose(): void
 }
 
+/** Reject anything that is not sink-shaped, naming what arrived. */
+function checkSink(sink: unknown): TelemetrySink {
+  if (typeof sink !== 'object' || sink === null) {
+    throw new Error(`telemetry sink must be an object, got ${sink === null ? 'null' : typeof sink}`)
+  }
+  const wants = (sink as TelemetrySink).wants
+  if (wants !== undefined && !Array.isArray(wants)) {
+    throw new Error(`telemetry sink 'wants' must be an array, got ${typeof wants}`)
+  }
+  return sink as TelemetrySink
+}
+
 /**
  * Consult every plugin's `telemetry` capability, collect the sinks, and —
  * only if at least one sink results — create a TelemetrySource and subscribe
@@ -49,23 +61,29 @@ export async function subscribeTelemetry(
   const sinks: TelemetrySink[] = extraSinks === undefined ? [] : [...extraSinks]
   for (const plugin of plugins) {
     if (plugin.telemetry === undefined) continue
-    let result
+    // A plugin's return value is user input, so it is checked here rather
+    // than trusted downstream: `createTelemetrySource` reads `.wants` off
+    // every sink immediately, so one `null` in the list used to abort the
+    // whole run with a raw TypeError before a single task had run. Staging
+    // into `accepted` keeps a partly-bad array all-or-nothing, and the shape
+    // check sits INSIDE the try so a throwing `wants` getter is caught too.
+    let accepted: TelemetrySink[]
     try {
-      result = await plugin.telemetry(ctx)
+      const result = await plugin.telemetry(ctx)
+      const list = result === undefined ? [] : Array.isArray(result) ? result : [result]
+      accepted = list.map((sink) => checkSink(sink))
     } catch (err) {
       ctx.warn(
         `[vx] plugin '${plugin.name}' telemetry failed to initialize; disabled for this run: ${err instanceof Error ? err.message : String(err)}`,
       )
       continue
     }
-    if (result === undefined) continue
-    if (Array.isArray(result)) sinks.push(...result)
-    else sinks.push(result)
+    sinks.push(...accepted)
   }
 
   if (sinks.length === 0) return undefined
 
-  const source = createTelemetrySource({ sinks, run })
+  const source = createTelemetrySource({ sinks, run, warn: (m) => ctx.warn(m) })
   const dispose = bus.subscribe(source.subscriber)
   let disposed = false
   return {

package/src/orchestrator/telemetry.ts

@@ -13,10 +13,11 @@
 // record; its `TelemetryContext` carries read-only metadata and NO mutable
 // run handle (no bus, no Cache, no RunRequest). There is no API path from a
 // sink back into scheduling/caching/exec — telemetry provably cannot change
-// what or how tasks run. Contrast `backend`/`cache`, which return objects
+// what or how tasks run. Contrast `cache`/`executor`, which return objects
 // core calls INTO; those are the behavior capabilities, kept separate.
 
 import type { OutputFingerprint, TaskStatus, VerifyVerdict } from '../graph/index.js'
+import { settleWithin, teardownTimeoutMs } from '../util/index.js'
 import type { RunEvent, RunEventSubscriber } from './events.js'
 
 /** Bumped when the record shape changes. Readers MUST check `v`. */
@@ -29,8 +30,9 @@ export type CacheSource = 'miss' | 'local' | 'remote' | 'none'
  * Map a task outcome status to its cache source. `success`/`failed` ran
  * (a miss as far as the cache read is concerned); the two cache-hit
  * statuses restored from local/remote; `skipped`/`aborted` never engaged
- * the cache. Pure function — the single place this mapping is decided, so
- * no consumer re-implements `status LIKE 'cache-hit%'`.
+ * the cache. Pure function — the single place this mapping is decided;
+ * `isCacheHit` below is derived from it rather than re-listing the two
+ * hit statuses, so there is one decision, not two that can disagree.
  */
 export function deriveCacheSource(status: TaskStatus): CacheSource {
   switch (status) {
@@ -47,6 +49,59 @@ export function deriveCacheSource(status: TaskStatus): CacheSource {
   }
 }
 
+/**
+ * Whether each status counts as a PASS. Written as a `Record<TaskStatus, …>`
+ * rather than a `new Set([...])` on purpose: a Record must name every member,
+ * so adding a status to the union is a COMPILE error HERE and the omission
+ * cannot ship. A Set of string literals has no such tripwire — it silently
+ * answers `false` for the new member, which is how ten hand-rolled copies of
+ * this vocabulary accumulated across core, cloud and the dashboard.
+ */
+const PASSES: Record<TaskStatus, boolean> = {
+  success: true,
+  'cache-hit': true,
+  'cache-hit-remote': true,
+  failed: false,
+  skipped: false,
+  aborted: false,
+}
+
+/**
+ * Every `TaskStatus`, at runtime. Read off `PASSES`'s keys, which the
+ * `Record<TaskStatus, …>` type guarantees is exactly the union — so this is
+ * derived, not a second list that can fall behind. Exported so a consumer that
+ * cannot import the type (a test asserting a copy of this vocabulary in
+ * another package agrees) can still enumerate the real set.
+ */
+export const TASK_STATUSES: readonly TaskStatus[] = Object.keys(PASSES) as TaskStatus[]
+
+const KNOWN_STATUSES: ReadonlySet<string> = new Set(TASK_STATUSES)
+
+/**
+ * Did the task pass? A cache hit counts — it produced the same result without
+ * spending the time, which is the whole point. `skipped` and `aborted` do NOT:
+ * neither finished on its own terms, so neither can vouch for anything.
+ *
+ * Takes `string`, not `TaskStatus`, because most callers hold a status that
+ * arrived over a wire or out of a database column. An unrecognised string
+ * reads as NOT passing — the safe direction, since the alternative is calling
+ * a run green on a status this build has never heard of.
+ */
+export function isPassStatus(status: string): boolean {
+  return PASSES[status as TaskStatus] === true
+}
+
+/**
+ * Did the task's result come out of the cache (either layer)? Derived from
+ * `deriveCacheSource` rather than re-listing the two hit statuses, so the two
+ * cannot disagree about what a hit is. Unknown strings read as not-a-hit.
+ */
+export function isCacheHit(status: string): boolean {
+  if (!KNOWN_STATUSES.has(status)) return false
+  const source = deriveCacheSource(status as TaskStatus)
+  return source === 'local' || source === 'remote'
+}
+
 /** Identifies which run a record belongs to + its captured context. Maps
  *  cleanly onto OTel CI/CD + VCS resource attributes. */
 export interface RunContextRecord {
@@ -71,6 +126,16 @@ export interface RunContextRecord {
   // git / CI / host — straight from run-context.ts.
   commitSha: string | null
   branch: string | null
+  /**
+   * The repository's DEFAULT (trunk) branch, when detectable (v2 additive).
+   * A run is a TRUNK run iff `branch === defaultBranch` (both non-null);
+   * everything else is PR / feature-branch work. Consumers use it to keep
+   * branch-experiment timings out of the shared scheduling baseline; null
+   * (undetectable) means "count all runs" — no regression. Required of a v2
+   * producer (every one emits it); a reader parsing an older v1 push, which
+   * predates the field, treats absent as null.
+   */
+  defaultBranch: string | null
   dirty: boolean | null
   ci: boolean
   ciProvider: string | null
@@ -94,6 +159,12 @@ export interface TaskTelemetry {
   hash?: string
   cpuMs?: number
   peakRssBytes?: number
+  /** Executor-reported placement (a worker id) — absent for local runs.
+   *  Additive-optional, no schema bump (the `attempts`/`verify` precedent). */
+  where?: string
+  /** `'deferred'` when the outputs stayed remote (`--download=none`).
+   *  Additive-optional, same no-bump precedent as `where`. */
+  outputs?: 'deferred'
   /** Total attempts when the task RETRIED (>1) — set only when `exec.retries`
    *  / `--retry` produced more than one attempt. A retried-then-passed task is
    *  flaky by definition; this is the telemetry-side flaky signal. */
@@ -122,7 +193,16 @@ export interface TaskTelemetry {
  * `task.log` records are large and OPT-IN (see `TelemetrySink.wants`).
  */
 export type TelemetryRecord =
-  | { v: number; kind: 'run.start'; run: RunContextRecord; total: number; ts: number }
+  | {
+      v: number
+      kind: 'run.start'
+      run: RunContextRecord
+      total: number
+      ts: number
+      /** The run's canonical start (epoch ms) — equals the summary's
+       *  `startedAt`; a sink derives per-task timing from it during the run. */
+      startedAt: number
+    }
   | {
       v: number
       kind: 'task.start'
@@ -166,6 +246,46 @@ export interface RunSummaryRecord {
   tasks: readonly TaskTelemetry[]
 }
 
+/**
+ * Assemble the canonical per-run summary from the per-task telemetry + the
+ * run-level timing/verdict. THE one place the `RunSummaryRecord` tallies are
+ * computed: both a local `run()` and the distributed controller build their
+ * `TaskTelemetry[]` (each via `deriveCacheSource`) and call this, so a
+ * distributed run and a local run produce byte-identical summaries and land in
+ * the same ingest. The per-task tallies (taskCount / failedCount /
+ * hitLocal|Remote|Count) derive from `tasks`; `totalDurationMs` (wall time) and
+ * `exitOk` (the run's overall verdict — which counts skipped/verify beyond the
+ * recorded task list) are run-level facts and are passed in.
+ */
+export function assembleRunSummary(
+  run: RunContextRecord,
+  tasks: readonly TaskTelemetry[],
+  timing: { startedAt: number; endedAt: number; totalDurationMs: number; exitOk: boolean },
+): RunSummaryRecord {
+  let failedCount = 0
+  let hitLocalCount = 0
+  let hitRemoteCount = 0
+  for (const t of tasks) {
+    if (t.status === 'failed') failedCount++
+    if (t.cacheSource === 'local') hitLocalCount++
+    else if (t.cacheSource === 'remote') hitRemoteCount++
+  }
+  return {
+    v: TELEMETRY_SCHEMA_VERSION,
+    run,
+    startedAt: timing.startedAt,
+    endedAt: timing.endedAt,
+    totalDurationMs: timing.totalDurationMs,
+    taskCount: tasks.length,
+    failedCount,
+    hitCount: hitLocalCount + hitRemoteCount,
+    hitLocalCount,
+    hitRemoteCount,
+    exitOk: timing.exitOk,
+    tasks,
+  }
+}
+
 /** A telemetry consumer. Observe-only: receives records, holds no run handle. */
 export interface TelemetrySink {
   readonly name?: string
@@ -180,7 +300,13 @@ export interface TelemetrySink {
   onRecord?(record: TelemetryRecord): void
   /** The per-run summary, at run:end. MUST return promptly. */
   onRunSummary?(summary: RunSummaryRecord): void
-  /** Drain buffered data. Awaited (time-bounded by the sink) at run:end. */
+  /**
+   * Drain buffered data. Awaited at run:end under a deadline — a sink that
+   * has not settled by then is abandoned and its buffered records are lost.
+   * Losing a slow sink's telemetry is strictly better than the alternative:
+   * `run()` never returns, so the cache never closes and `vx` exits 0 on a
+   * failed run.
+   */
   flush?(): Promise<void>
 }
 
@@ -199,7 +325,7 @@ export interface TelemetrySource {
   readonly subscriber: RunEventSubscriber
   /** Fan the per-run summary to every sink's `onRunSummary` (crash-isolated). */
   emitSummary(summary: RunSummaryRecord): void
-  /** Await every sink's `flush()` (each crash-isolated). */
+  /** Await every sink's `flush()` (each crash-isolated, all time-bounded). */
   flush(): Promise<void>
 }
 
@@ -224,12 +350,27 @@ const DEFAULT_KINDS: ReadonlyArray<TelemetryRecord['kind']> = [
 export function createTelemetrySource(args: {
   sinks: readonly TelemetrySink[]
   run: RunContextRecord
+  /** Where a dropped-flush notice goes; absent = stay silent. */
+  warn?: (message: string) => void
 }): TelemetrySource {
-  const { sinks, run } = args
+  const { sinks, run, warn } = args
   const runId = run.runId
   // A sink is disabled the first time it throws — its name (or index) goes
-  // here and it's skipped for the rest of the run.
+  // here and it's skipped for the rest of the run, FLUSH INCLUDED. Flushing a
+  // disabled sink would write a buffer that is incomplete by construction: it
+  // stopped being fed records at the moment it threw, so the export would look
+  // like a run that simply had fewer tasks.
   const disabled = new Set<TelemetrySink>()
+  // Never silently. The standing rule is that a never-fail path must still
+  // WARN — telemetry that vanishes without a word is indistinguishable from
+  // telemetry nobody configured.
+  const disable = (sink: TelemetrySink, hook: string, err: unknown): void => {
+    if (disabled.has(sink)) return
+    disabled.add(sink)
+    warn?.(
+      `[vx] telemetry sink '${sink.name}' threw in ${hook}; disabled for this run: ${err instanceof Error ? err.message : String(err)}`,
+    )
+  }
   // Precompute which sinks want each kind, so per-event fan-out is a plain
   // array walk with no per-record `wants` scanning.
   const wantsLog = sinks.some((s) => (s.wants ?? DEFAULT_KINDS).includes('task.log'))
@@ -241,8 +382,8 @@ export function createTelemetrySource(args: {
       if (!kinds.includes(record.kind)) continue
       try {
         sink.onRecord(record)
-      } catch {
-        disabled.add(sink)
+      } catch (err) {
+        disable(sink, 'onRecord', err)
       }
     }
   }
@@ -258,6 +399,7 @@ export function createTelemetrySource(args: {
           run,
           total: event.info.total,
           ts,
+          startedAt: event.info.startedAtMs ?? ts,
         })
         return
       case 'task:start': {
@@ -309,6 +451,8 @@ export function createTelemetrySource(args: {
         if (outcome.hash !== undefined) rec.hash = outcome.hash
         if (outcome.cpuMs !== undefined) rec.cpuMs = outcome.cpuMs
         if (outcome.peakRssBytes !== undefined) rec.peakRssBytes = outcome.peakRssBytes
+        if (outcome.where !== undefined) rec.where = outcome.where
+        if (outcome.outputs !== undefined) rec.outputs = outcome.outputs
         if (outcome.attempts !== undefined) rec.attempts = outcome.attempts
         if (outcome.verify !== undefined) rec.verify = outcome.verify
         if (outcome.outputFp !== undefined) rec.outputFp = outcome.outputFp
@@ -337,22 +481,38 @@ export function createTelemetrySource(args: {
         if (disabled.has(sink) || sink.onRunSummary === undefined) continue
         try {
           sink.onRunSummary(summary)
-        } catch {
-          disabled.add(sink)
+        } catch (err) {
+          disable(sink, 'onRunSummary', err)
         }
       }
     },
     async flush(): Promise<void> {
-      await Promise.all(
-        sinks.map(async (sink) => {
-          if (sink.flush === undefined) return
-          try {
-            await sink.flush()
-          } catch {
-            // a sink's flush failure can never break the run
-          }
-        }),
+      const ms = teardownTimeoutMs()
+      // Bounded, for the same reason plugin teardown is bounded in
+      // plugin-host.ts: run() awaits this BEFORE closeCache() and before it
+      // returns, and bin.ts is `process.exit(await run(...))`. A sink whose
+      // flush never settles therefore drains the event loop with no exit code
+      // pending — Bun exits 0 and a FAILED run reports green, with the cache's
+      // accessed_at bumps and every later plugin's teardown lost with it.
+      // Sinks race concurrently, so each still gets the whole budget.
+      const settled = await settleWithin(
+        Promise.all(
+          sinks.map(async (sink) => {
+            if (disabled.has(sink) || sink.flush === undefined) return
+            try {
+              await sink.flush()
+            } catch (err) {
+              // A flush failure can never break the run — but it still costs
+              // this sink its export, so say so rather than dropping it.
+              warn?.(
+                `[vx] telemetry sink '${sink.name}' failed to flush: ${err instanceof Error ? err.message : String(err)}`,
+              )
+            }
+          }),
+        ),
+        ms,
       )
+      if (!settled) warn?.(`[vx] telemetry flush timed out after ${ms}ms; buffered records lost`)
     },
   }
 }