@vzn/vx 0.0.16 → 0.0.21

package/src/orchestrator/download-policy.ts

@@ -0,0 +1,171 @@
+// `--download` policy: which tasks may leave their outputs in the remote
+// store instead of landing them on this machine, and which must not.
+//
+// See docs/design/download-policy-cas-cache-2026-08.md. Two decisions live
+// here, both made ONCE per run at plan time (never per attempt), so
+// `--dry` can show them and the scheduler never re-derives them.
+
+import { isGroupTask, type TaskNode } from '../graph/index.js'
+import { staticPrefix } from '../util/index.js'
+
+/** Where a task's outputs go. `never` is `exec.remote: 'only'`. */
+export type DownloadMode = 'eager' | 'deferred' | 'never'
+
+/**
+ * Deferral eligibility: the tasks whose outputs NO key in this run could
+ * observe on disk. Returns the INELIGIBLE ids mapped to the reason, so a
+ * downgrade can name itself in `--dry`.
+ *
+ * The channel this closes is narrow and specific. A dependent's key folds
+ * an upstream's KEY, never its output content (pure-input transitive
+ * hashing), so deferral cannot move a dependent's key that way. The one
+ * real channel is a task whose `cache.inputs` globs can MATCH a producer's
+ * declared outputs on disk: then its key differs by whether the bytes
+ * arrived.
+ *
+ * DELIBERATELY TIGHTER than "any task sharing the producer's project"
+ * (the shape the design doc first sketched): that rule fires whenever a
+ * sibling reads the project at all, which is every ordinary workspace —
+ * `test` reads `src/**` while `build` writes `dist/**` — and would leave
+ * `--download=none` with nothing to defer. Comparing the globs' static
+ * prefixes answers the question actually being asked, and stays
+ * conservative in every direction that matters: a leading wildcard yields
+ * `.` and reaches everything, a cacheable task with no declared `files`
+ * is treated as reading its whole project, and `workspaceFiles` on either
+ * side ignores project boundaries — all three force ineligible.
+ */
+export function deferralEligibility(nodes: Map<string, TaskNode>): Map<string, string> {
+  const ineligible = new Map<string, string>()
+
+  const readersByProject = new Map<string, { taskId: string; prefixes: string[] }[]>()
+  let workspaceReader: string | undefined
+  let runtimeReader: string | undefined
+  for (const n of nodes.values()) {
+    const cache = n.config.cache
+    if (cache === undefined) continue
+    if ((cache.inputs?.workspaceFiles?.length ?? 0) > 0) workspaceReader ??= n.id
+    // A `runtime` input is a SHELL COMMAND whose reads are unknowable —
+    // the same reason vx refuses to infer inputs by tracing. It can `cat` a
+    // producer's output (or path-escape its project to do it), and its
+    // stdout is folded into the key, so its answer would differ by whether
+    // the bytes were fetched. Deferral makes that sharper than it already
+    // was: it deliberately skips the output clean, so a stale prior build
+    // is exactly what such a command would sample. Nothing defers in a run
+    // that declares one — worse than nothing would be a key that moves with
+    // a transfer flag.
+    if (
+      (cache.inputs?.runtime?.length ?? 0) > 0 ||
+      (cache.inputs?.workspaceRuntime?.length ?? 0) > 0
+    ) {
+      runtimeReader ??= n.id
+    }
+    const files = cache.inputs?.files
+    const prefixes = files === undefined || files.length === 0 ? ['.'] : files.map(staticPrefix)
+    const list = readersByProject.get(n.projectName)
+    if (list) list.push({ taskId: n.id, prefixes })
+    else readersByProject.set(n.projectName, [{ taskId: n.id, prefixes }])
+  }
+
+  for (const n of nodes.values()) {
+    const outputs = n.config.cache?.outputs
+    if ((outputs?.workspaceFiles?.length ?? 0) > 0) {
+      ineligible.set(
+        n.id,
+        'declares cache.outputs.workspaceFiles — root-anchored outputs can land where any project-relative input reads',
+      )
+      continue
+    }
+    const files = outputs?.files ?? []
+    if (files.length === 0) continue
+    if (runtimeReader !== undefined) {
+      ineligible.set(
+        n.id,
+        `${runtimeReader} declares a cache.inputs.runtime command, whose reads cannot be bounded`,
+      )
+      continue
+    }
+    if (workspaceReader !== undefined) {
+      ineligible.set(
+        n.id,
+        `${workspaceReader} declares cache.inputs.workspaceFiles, which can read any project`,
+      )
+      continue
+    }
+    const outPrefixes = files.map(staticPrefix)
+    // Project boundaries are hard, so only a SAME-project reader can reach a
+    // project-relative output. A task never observes its own outputs — they
+    // are excluded from its own key by construction.
+    for (const reader of readersByProject.get(n.projectName) ?? []) {
+      if (reader.taskId === n.id) continue
+      const clash = reader.prefixes.find((ip) => outPrefixes.some((op) => prefixesOverlap(ip, op)))
+      if (clash !== undefined) {
+        ineligible.set(
+          n.id,
+          `${reader.taskId} reads ${clash === '.' ? 'the whole project' : clash} with cache.inputs.files`,
+        )
+        break
+      }
+    }
+  }
+  return ineligible
+}
+
+/** Could two glob prefixes, anchored at the same dir, name overlapping paths? */
+function prefixesOverlap(a: string, b: string): boolean {
+  if (a === '.' || b === '.' || a === '' || b === '' || a === '/' || b === '/') return true
+  if (a === b) return true
+  return a.startsWith(`${b}/`) || b.startsWith(`${a}/`)
+}
+
+/**
+ * The effective per-task download mode. `all` (the default) reproduces
+ * today's behaviour exactly: every task eager.
+ */
+export function resolveDownloadModes(args: {
+  nodes: Map<string, TaskNode>
+  policy: 'all' | 'toplevel' | 'none'
+  /** Task ids placed on a LOCAL executor — they write in place. */
+  localPlaced: ReadonlySet<string>
+  /** Task ids with `exec.remote: 'only'`. */
+  remoteOnly: ReadonlySet<string>
+}): { modeOf: Map<string, DownloadMode>; downgrades: Map<string, string> } {
+  const modeOf = new Map<string, DownloadMode>()
+  const downgrades = new Map<string, string>()
+  const ineligible = args.policy === 'all' ? new Map() : deferralEligibility(args.nodes)
+
+  for (const n of args.nodes.values()) {
+    if (isGroupTask(n)) continue
+    if (args.remoteOnly.has(n.id)) {
+      modeOf.set(n.id, 'never')
+      continue
+    }
+    if (args.policy === 'all' || args.localPlaced.has(n.id)) {
+      modeOf.set(n.id, 'eager')
+      continue
+    }
+    // `toplevel` = the outputs you ASKED for come home; intermediates stay
+    // remote. Decided here rather than by materialising at run end: eager
+    // materialisation rides each task's own completion, overlapped with the
+    // rest of the run, where a run-end batch would serialise every download
+    // after the last task finishes.
+    //
+    // SURFACED counts as asked-for. A requested GROUP has no outputs of its
+    // own — `markSurfacedDeps` marks the real tasks it chains — so keying on
+    // `requested` alone made `vx run ci --download=toplevel` (ci being a
+    // group) bring home NOTHING, which is the one thing the mode exists to
+    // avoid. run.ts already pairs the two flags for the same reason when it
+    // decides which persistent children to keep alive.
+    if (args.policy === 'toplevel' && (n.requested === true || n.surfaced === true)) {
+      modeOf.set(n.id, 'eager')
+      continue
+    }
+    const reason = ineligible.get(n.id)
+    if (reason !== undefined) {
+      modeOf.set(n.id, 'eager')
+      downgrades.set(n.id, reason)
+      continue
+    }
+    modeOf.set(n.id, 'deferred')
+  }
+  return { modeOf, downgrades }
+}

package/src/orchestrator/events.ts

@@ -18,6 +18,11 @@ export interface RunStartInfo {
   requestedCount?: number
   /** Run banner context so a live footer can match the final summary. */
   context?: RunContext
+  /** The run's canonical start (epoch ms) — the SAME value the end-of-run
+   *  summary reports as `startedAt`. A telemetry sink needs it during the run
+   *  (e.g. to derive per-task started_at consistently with the batch summary
+   *  for idempotent incremental ingest). */
+  startedAtMs?: number
 }
 
 /**
@@ -54,7 +59,7 @@ export function createEventBus(): EventBus {
       // depends on it), so emit must never reorder or defer.
       //
       // A throwing subscriber must NEVER break the run — the whole point
-      // of the bus is that a surface (the devframe dev server, a future
+      // of the bus is that a surface (an embedder's renderer, a future
       // TUI) is isolated from execution. We swallow per-subscriber so one
       // bad surface can't crash the user's build (mirrors the deleted
       // Observer's makeSafeObserver contract). The terminal renderer is
@@ -133,9 +138,9 @@ export function terminalSubscriber(sink: Logger): RunEventSubscriber {
 
 /**
  * A bus subscriber that projects each event to its `WireEvent` and hands
- * it to `send` — the shared run→consumer forwarding path (the `vx dev`
- * hub frames each as NDJSON; the `vx serve` protocol wraps each in an
- * envelope). Group-task start/complete events are dropped (no command,
+ * it to `send` — the shared run→consumer forwarding path for whatever an
+ * embedder builds on it (an NDJSON stream, an enveloped protocol; core
+ * ships neither). Group-task start/complete events are dropped (no command,
  * pure scheduling noise). `run()` emits run:end twice (normal + finally)
  * plus trailing summary status lines; we forward the run once and then go
  * quiet, so a consumer sees exactly one terminal frame per run. `send`
@@ -146,9 +151,17 @@ export function wireForwarder(send: (event: WireEvent) => void): RunEventSubscri
   // run() emits run:end TWICE (normal + finally), with the summary footer
   // (run:status lines) emitted in BETWEEN. Drop the duplicate run:end, but
   // keep forwarding everything else — including those post-run:end status
-  // lines, so a consumer that renders them (the serve client) gets the
-  // footer. A consumer that ignores run:status (the dev hub) is unaffected.
+  // lines, so a consumer that renders them gets the footer, and one that
+  // ignores run:status is unaffected.
   let endForwarded = false
+  // Which tasks the consumer has been told about. A skipped task never
+  // reaches the scheduler's onStart, so its completion arrives with no
+  // preceding task:start — and `createWireRenderer` resolves a completion's
+  // node from the start it recorded, so it would drop the task entirely
+  // while the forwarded footer still counted it. Synthesize the start here,
+  // where the live TaskNode is in hand, so the projection stays full
+  // fidelity (real requested/surfaced/command) instead of a stand-in.
+  const started = new Set<string>()
   return (event) => {
     if (event.kind === 'run:end') {
       if (endForwarded) return
@@ -158,12 +171,17 @@ export function wireForwarder(send: (event: WireEvent) => void): RunEventSubscri
       isGroupTask(event.node)
     ) {
       return
+    } else if (event.kind === 'task:start') {
+      started.add(event.node.id)
+    } else if (event.kind === 'task:complete' && !started.has(event.node.id)) {
+      started.add(event.node.id)
+      send({ kind: 'task:start', task: projectNode(event.node) })
     }
     send(toWireEvent(event))
   }
 }
 
-// --- Serializable projections (the off-thread / devframe wire form) ----
+// --- Serializable projections (the off-thread wire form) ---------------
 //
 // The in-process bus carries live refs; crossing a worker postMessage or
 // an RPC needs a JSON/structured-clone-safe form. Two concrete blockers
@@ -195,9 +213,26 @@ export interface OutcomeView {
   status: TaskOutcome['status']
   exitCode: number
   durationMs: number
+  /**
+   * Group node (no `exec`) — organizational, not work. `TaskView` carries
+   * this too, but a consumer that only holds outcomes (the `--report`
+   * writer) has no node to ask, and counting groups as executed tasks is
+   * exactly how it drifted from the terminal and `--summarize`.
+   */
+  isGroup?: boolean
+  /** Executed with no `cache` block: never consulted the cache, not a miss. */
+  noCache?: boolean
+  /** Cache hits only: the exec time the entry was stored with — the work
+   *  this hit skipped, as opposed to `durationMs`, the restore it cost. */
+  storedDurationMs?: number
+  /** Cache hits only: what the producing execution used (see `TaskOutcome`). */
+  storedCpuMs?: number
+  storedPeakRssBytes?: number
   hash?: string
   cpuMs?: number
   peakRssBytes?: number
+  /** How long an `admit` policy held the task with a worker free (see `TaskOutcome`). */
+  admissionHeldMs?: number
   restored?: boolean
   sandboxViolations?: number
   sandboxViolationLines?: string[]
@@ -206,6 +241,29 @@ export interface OutcomeView {
   wallclockEndNs?: string
 }
 
+/**
+ * The one outcome vocabulary every renderer prints — the framed block
+ * header and footer, the focused one-liner, the hash-only audit line,
+ * the verbose summary table: `success` / `restored-local` /
+ * `restored-remote` / `up-to-date` / `failed` / `skipped`. A hit that
+ * materialized nothing is up-to-date whichever layer answered.
+ */
+export function outcomeWord(o: Pick<OutcomeView, 'status' | 'restored'>): string {
+  switch (o.status) {
+    case 'cache-hit':
+      return o.restored === false ? 'up-to-date' : 'restored-local'
+    case 'cache-hit-remote':
+      return o.restored === false ? 'up-to-date' : 'restored-remote'
+    default:
+      return o.status
+  }
+}
+
+/** `outcomeWord` with the exit code a failure carries: `failed (exit 9)`. */
+export function outcomeLabel(o: Pick<OutcomeView, 'status' | 'restored' | 'exitCode'>): string {
+  return o.status === 'failed' ? `failed (exit ${o.exitCode})` : outcomeWord(o)
+}
+
 export function projectNode(node: TaskNode): TaskView {
   const view: TaskView = {
     id: node.id,
@@ -227,9 +285,15 @@ export function projectOutcome(outcome: TaskOutcome): OutcomeView {
     exitCode: outcome.exitCode,
     durationMs: outcome.durationMs,
   }
+  if (isGroupTask(outcome.node)) view.isGroup = true
+  else if (outcome.node.config.cache === undefined) view.noCache = true
+  if (outcome.storedDurationMs !== undefined) view.storedDurationMs = outcome.storedDurationMs
+  if (outcome.storedCpuMs !== undefined) view.storedCpuMs = outcome.storedCpuMs
+  if (outcome.storedPeakRssBytes !== undefined) view.storedPeakRssBytes = outcome.storedPeakRssBytes
   if (outcome.hash !== undefined) view.hash = outcome.hash
   if (outcome.cpuMs !== undefined) view.cpuMs = outcome.cpuMs
   if (outcome.peakRssBytes !== undefined) view.peakRssBytes = outcome.peakRssBytes
+  if (outcome.admissionHeldMs !== undefined) view.admissionHeldMs = outcome.admissionHeldMs
   if (outcome.restored !== undefined) view.restored = outcome.restored
   if (outcome.sandboxViolations !== undefined) view.sandboxViolations = outcome.sandboxViolations
   if (outcome.sandboxViolationLines !== undefined)
@@ -242,7 +306,7 @@ export function projectOutcome(outcome: TaskOutcome): OutcomeView {
 
 /**
  * The serializable form of a RunEvent — JSON / structured-clone safe, the
- * wire contract a non-local consumer (devframe, `vx serve` client) reads.
+ * wire contract a non-local consumer (an embedder's client) reads.
  * `task:start` carries the full `TaskView` so a consumer can rebuild a
  * node-shaped object incrementally (no upfront table needed); later
  * task events reference the id, which the consumer already holds.