@vzn/vx 0.0.645 → 0.0.648

package/src/orchestrator/output-host.ts

@@ -4,10 +4,12 @@
 
 import { terminalSubscriber, type RunEventSubscriber } from '../report/index.js'
 import type { Logger } from '../report/index.js'
+import { settleWithin, teardownTimeoutMs, thrownMessage } from '../util/index.js'
 import type { OutputContext, VxPlugin } from './plugin.js'
 
 const REQUIRED = ['status', 'taskStdout', 'taskStderr', 'taskComplete'] as const
 const OPTIONAL = [
+  'warn',
   'runStart',
   'taskStart',
   'runEnd',
@@ -26,14 +28,17 @@ function checkLogger(value: unknown): Logger {
   const missing = REQUIRED.filter((k) => typeof o[k] !== 'function')
   if (missing.length > 0) throw new Error(`output Logger has no ${missing.join(', ')}`)
   const wrong = OPTIONAL.filter((k) => o[k] !== undefined && typeof o[k] !== 'function')
-  if (wrong.length > 0) throw new Error(`output Logger's ${wrong.join(', ')} must be functions`)
+  if (wrong.length > 0)
+    throw new Error(
+      `output Logger's ${wrong.join(', ')} must be ${wrong.length === 1 ? 'a function' : 'functions'}`,
+    )
   return value as Logger
 }
 
 /**
  * The first plugin, in declaration order, whose `output` returns a Logger.
- * A hook that throws or returns something else is warned about and passed
- * over: a renderer must never fail a run.
+ * A hook that throws, returns something else or does not answer in time
+ * is warned about and passed over: a renderer must never fail a run.
  */
 export async function claimOutput(
   plugins: readonly VxPlugin[],
@@ -42,12 +47,20 @@ export async function claimOutput(
   for (const plugin of plugins) {
     if (plugin.output === undefined) continue
     try {
-      const logger = await plugin.output(ctx)
+      // Bounded as telemetry's hook is: one that never settled held the
+      // run before its first task until the loop drained, exit 1 (item 921).
+      let logger: Logger | undefined
+      const ms = teardownTimeoutMs()
+      const arrived = await settleWithin(
+        Promise.resolve(plugin.output(ctx)).then((l) => {
+          logger = l
+        }),
+        ms,
+      )
+      if (!arrived) throw new Error(`did not answer within ${ms} ms`)
       if (logger !== undefined) return { logger: checkLogger(logger), owner: plugin.name }
     } catch (err) {
-      ctx.warn(
-        `[vx] plugin '${plugin.name}' output: ${err instanceof Error ? err.message : String(err)}; vx renders the run`,
-      )
+      ctx.warn(`[vx] plugin '${plugin.name}' output: ${thrownMessage(err)}; vx renders the run`)
     }
   }
   return undefined
@@ -62,19 +75,35 @@ export function guardedOutput(
   claimed: { logger: Logger; owner: string },
   fallback: RunEventSubscriber,
 ): RunEventSubscriber {
-  const render = terminalSubscriber(claimed.logger)
   let failed = false
+  const fail = (err: unknown): void => {
+    if (failed) return
+    failed = true
+    fallback({
+      kind: 'run:status',
+      warn: true,
+      line: `[vx] plugin '${claimed.owner}' output failed: ${thrownMessage(err)}; vx renders the rest of the run`,
+    })
+  }
+  // An `async` method's rejection reached no one: an unhandled rejection
+  // ended the run with a stack, exit 1, after its tasks had passed. It
+  // fails the Logger as a throw does; the event it was handed stays its.
+  const logger = Object.create(claimed.logger) as Logger
+  for (const k of [...REQUIRED, ...OPTIONAL, 'warn'] as const) {
+    const method = claimed.logger[k] as ((...args: unknown[]) => unknown) | undefined
+    if (typeof method !== 'function') continue
+    ;(logger as unknown as Record<string, unknown>)[k] = (...args: unknown[]): void => {
+      const ret = method.apply(claimed.logger, args)
+      if (ret instanceof Promise) ret.catch(fail)
+    }
+  }
+  const render = terminalSubscriber(logger)
   return (event) => {
     if (failed) return fallback(event)
     try {
       render(event)
     } catch (err) {
-      failed = true
-      fallback({
-        kind: 'run:status',
-        warn: true,
-        line: `[vx] plugin '${claimed.owner}' output failed: ${err instanceof Error ? err.message : String(err)}; vx renders the rest of the run`,
-      })
+      fail(err)
       fallback(event)
     }
   }

package/src/orchestrator/output-log.ts

@@ -4,16 +4,19 @@
 // RS (U+001E) + `e` / `o` where the stream switches to stderr / stdout, and
 // a literal RS doubled. Output that never touched stderr is its own text.
 
-import { BoundedCapture, type CapturedChunk } from '../exec/index.js'
+import { type BoundedCapture, type CapturedChunk, isDroppedOutputChunk } from '../exec/index.js'
+import { MASKED, type SecretMask } from '../util/index.js'
 
 const RS = '\x1e'
 
-/** `chunks` as one stored string; it starts on stdout. */
-export function encodeOutputLog(chunks: readonly CapturedChunk[]): string {
+/**
+ * `chunks` as one stored string; it starts on stdout. `mask` (the task's
+ * secrets) is applied to each stream's whole text (`maskStreams`).
+ */
+export function encodeOutputLog(chunks: readonly CapturedChunk[], mask?: SecretMask): string {
   let out = ''
   let err = false
-  for (const c of chunks) {
-    if (c.text.length === 0) continue
+  for (const c of maskStreams(chunks, mask)) {
     if (c.err !== err) {
       out += RS + (c.err ? 'e' : 'o')
       err = c.err
@@ -23,6 +26,76 @@ export function encodeOutputLog(chunks: readonly CapturedChunk[]): string {
   return out
 }
 
+/**
+ * `encodeOutputLog` with `mask` applied, for a capture of raw output: a
+ * value the capture's dropped middle cut left a piece either side, which
+ * only `maskCut` sees, as `maskCaptured` does for core's own capture.
+ */
+export function maskedOutputLog(chunks: readonly CapturedChunk[], mask: SecretMask): string {
+  const at = chunks.findIndex(isDroppedOutputChunk)
+  if (at === -1) return encodeOutputLog(chunks, mask)
+  const before = chunks.slice(0, at)
+  const tail = encodeOutputLog(chunks.slice(at + 1))
+  // The dropped line is on stdout, where the tail's own log starts.
+  const lead = tail.startsWith(RS + 'e') ? RS + 'e' : ''
+  const [h, t] = mask.maskCut(encodeOutputLog(before), tail.slice(lead.length))
+  const back = before.findLast((c) => c.text.length > 0)?.err === true ? RS + 'o' : ''
+  return h + back + chunks[at]!.text + lead + t
+}
+
+/**
+ * Each stream's run of adjacent non-empty chunks joined, with `mask`
+ * applied to each stream's text as a whole: a value is masked where it
+ * starts and its rest dropped from the runs it reaches. The live stream
+ * masks chunk by chunk and lets a held piece go after a pause, so a value
+ * written in two pieces reached the capture in two chunks, and a write to
+ * the other stream between them made them two runs. A hit replays each
+ * stream to its own descriptor, so `vx run > log` joined them again.
+ */
+export function maskStreams(
+  chunks: readonly CapturedChunk[],
+  mask: SecretMask | undefined,
+): CapturedChunk[] {
+  if (mask === undefined) return chunks.filter((c) => c.text.length > 0)
+  const runs: CapturedChunk[] = []
+  for (const c of chunks) {
+    if (c.text.length === 0) continue
+    const last = runs.at(-1)
+    if (last !== undefined && last.err === c.err) last.text += c.text
+    else runs.push({ text: c.text, err: c.err })
+  }
+  for (const err of [false, true]) {
+    const own = runs.filter((r) => r.err === err)
+    const spans = mask.spans(own.map((r) => r.text).join(''))
+    if (spans.length === 0) continue
+    let at = 0
+    let i = 0
+    for (const r of own) {
+      const end = at + r.text.length
+      let text = ''
+      let from = at
+      for (; i < spans.length && spans[i]![0] < end; i++) {
+        const [s, e] = spans[i]!
+        if (s >= from) text += r.text.slice(from - at, s - at) + MASKED
+        from = Math.max(from, Math.min(e, end))
+        // A value that runs past this run is cut from the next one too.
+        if (e > end) break
+      }
+      r.text = text + r.text.slice(from - at)
+      at = end
+    }
+  }
+  // A run emptied whole leaves its neighbours, the other stream, adjacent.
+  const kept: CapturedChunk[] = []
+  for (const r of runs) {
+    if (r.text.length === 0) continue
+    const last = kept.at(-1)
+    if (last !== undefined && last.err === r.err) last.text += r.text
+    else kept.push(r)
+  }
+  return kept
+}
+
 /** The chunks `encodeOutputLog` wrote, adjacent same-stream text joined. */
 export function decodeOutputLog(log: string): CapturedChunk[] {
   if (!log.includes(RS)) return log.length === 0 ? [] : [{ text: log, err: false }]
@@ -58,9 +131,16 @@ export const FAILED_OUTPUT_HEAD_CHARS = 8 * 1024
 export const FAILED_OUTPUT_TAIL_CHARS = 56 * 1024
 
 /** A failed attempt's capture, re-bounded to the failure's limits when it is the entry's. */
-export function failedOutputLog(capture: BoundedCapture, entrySized: boolean): string {
-  if (!entrySized) return encodeOutputLog(capture.chunks())
-  const small = new BoundedCapture(FAILED_OUTPUT_HEAD_CHARS, FAILED_OUTPUT_TAIL_CHARS)
-  for (const c of capture.chunks()) small.push(c.text, c.err)
-  return encodeOutputLog(small.chunks())
+export function failedOutputLog(
+  capture: BoundedCapture,
+  entrySized: boolean,
+  mask?: SecretMask,
+): string {
+  if (!entrySized) return encodeOutputLog(capture.chunks(), mask)
+  // Masked before the re-bound: its cut must not split a value first.
+  return encodeOutputLog(
+    capture
+      .rebound(FAILED_OUTPUT_HEAD_CHARS, FAILED_OUTPUT_TAIL_CHARS, (c) => maskStreams(c, mask))
+      .chunks(),
+  )
 }

package/src/orchestrator/persistent-task.ts

@@ -4,15 +4,17 @@
 
 import type { ExecConfig } from '../config.js'
 import {
+  lineEnds,
   PersistentReadyError,
   releaseBridges,
   runPersistent,
+  sandboxReads,
   signalExitCode,
   withForwardArgs,
   wrapSandboxedCommand,
 } from '../exec/index.js'
 import type { TaskOutcome } from '../graph/index.js'
-import { maskedEmitter, secretMask, testTimerMs } from '../util/index.js'
+import { maskedStreams, secretMask, testTimerMs } from '../util/index.js'
 import { warnOf } from '../report/index.js'
 import type { ExecuteArgs } from './execute-task.js'
 import {
@@ -25,13 +27,43 @@ import {
   undeclaredWriteReach,
   untouchedPlaceholderLine,
 } from './sandbox-request.js'
+import { shellVerdict } from './shell-verdict.js'
 import { forwardedSignal, terminateChildren } from './signals.js'
 import { computeTaskHash } from './task-hash.js'
-import { forgetUndeclaredWrites, isAborted, taskEnv, taskTimeoutMs } from './task-process.js'
+import {
+  forgetUndeclaredWrites,
+  isAborted,
+  taskBinDirs,
+  taskEnv,
+  taskTimeoutMs,
+} from './task-process.js'
 
 /** How long a server may take to match `readyWhen` before vx says it waits. */
 const READY_NOTICE_MS = 10_000
 
+/**
+ * A server is keyed on its inputs like any task, so a cached dependant
+ * behind a dev server re-runs when the server's sources change; with none
+ * it has no key and the dependant always runs (A-17). A server the watch
+ * loop keeps up is keyed too: its dependant folds the key either way.
+ */
+export function serverKey(args: ExecuteArgs): Promise<string | undefined> | string | undefined {
+  if (args.noDependants === true) return undefined
+  return (
+    args.upfrontKey ??
+    computeTaskHash({
+      node: args.node,
+      upstream: args.upstream,
+      workspaceRoot: args.workspaceRoot,
+      cache: args.cache,
+      forwardArgs: args.forwardArgs,
+      nestedProjectDirs: args.nestedProjectDirs,
+      ...(args.gitFilesCache !== undefined ? { gitFilesCache: args.gitFilesCache } : {}),
+      ...(args.hashCache !== undefined ? { hashCache: args.hashCache } : {}),
+    })
+  )
+}
+
 /**
  * Persistent task: dev server / file watcher / daemon. Spawn, wait
  * for ready (regex match or immediate), stash the subprocess in the
@@ -55,23 +87,22 @@ export async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutc
   const effectiveForwardArgs = node.requested ? (args.forwardArgs ?? []) : []
   const env = taskEnv(node, step, args.workspaceRoot)
   const wallclockStartNs = process.hrtime.bigint() - args.runStartHrTimeNs
-  // Keyed on its inputs like any task, so a cached dependant behind a dev
-  // server re-runs when the server's sources change; with none it has no
-  // key and the dependant always runs (A-17).
-  const hash =
-    args.noDependants === true
-      ? undefined
-      : (args.upfrontKey ??
-        (await computeTaskHash({
-          node,
-          upstream: args.upstream,
-          workspaceRoot: args.workspaceRoot,
-          cache: args.cache,
-          forwardArgs: args.forwardArgs,
-          nestedProjectDirs: args.nestedProjectDirs,
-          ...(args.gitFilesCache !== undefined ? { gitFilesCache: args.gitFilesCache } : {}),
-          ...(args.hashCache !== undefined ? { hashCache: args.hashCache } : {}),
-        })))
+  const hash = await serverKey(args)
+
+  // Every dependant starts it where it runs (`ServerSpec`): nothing to spawn
+  // here, and nothing that ran (2026-10-22: nothing runs on a
+  // distributed run's orchestrator).
+  if (args.carried === true)
+    return {
+      node,
+      status: 'success',
+      exitCode: 0,
+      durationMs: 0,
+      carried: true,
+      ...(hash === undefined ? {} : { hash }),
+      wallclockStartNs,
+      wallclockEndNs: wallclockStartNs,
+    }
 
   // The args after `--` reach a server as they reach any task. A readyWhen
   // server once got none — "so the matcher sees the unmodified output",
@@ -92,6 +123,7 @@ export async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutc
   // server that dies before readiness is told the directory spelling — the
   // same trap and the same answer as a one-shot task's (sandbox-request.ts).
   let placeholders: Placeholder[] = []
+  let serverSandbox: Awaited<ReturnType<typeof sandboxRequestFor>>['sandbox'] | undefined
   if (step.sandbox !== undefined) {
     await args.armSandbox?.()
     // A persistent task never declares `cache` (the loader refuses it), so
@@ -105,6 +137,7 @@ export async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutc
       args.cacheDir,
     )
     placeholders = sb.placeholders
+    serverSandbox = sb.sandbox
     // A wrap that refuses (a port the host holds) leaves the server unspawned
     // and the placeholders vx made for it in the project for every later run.
     const wrapped = await wrapSandboxedCommand({
@@ -125,16 +158,21 @@ export async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutc
   // is matched on the raw chunks, before this. Its held tail is flushed by
   // the idle timer: a server never ends on the way to a flush.
   const serverSecrets = secretMask([process.env, env, step.env?.define], step.env?.secret)
-  const serverOut = serverSecrets && maskedEmitter(serverSecrets, (t) => log.taskStdout(node, t))
-  const serverErr = serverSecrets && maskedEmitter(serverSecrets, (t) => log.taskStderr(node, t))
   // A server writes until the run stops it: its window stays open.
   args.commandWindows?.open(node.id, node.projectDir, commandWriteReach(node, args.workspaceRoot))
+  // Whether the server's output ends mid-line, for vx's never-ready note.
+  const ends = lineEnds(
+    (t) => log.taskStdout(node, t),
+    (t) => log.taskStderr(node, t),
+  )
+  // Masked as one: a stream's held tail goes out before the other's next chunk.
+  const serverMasked = serverSecrets && maskedStreams(serverSecrets, ends.onStdout!, ends.onStderr!)
   const persistentOpts: Parameters<typeof runPersistent>[0] = {
     command,
     cwd: node.projectDir,
     env,
-    onStdout: serverOut ? (chunk) => serverOut.push(chunk) : (chunk) => log.taskStdout(node, chunk),
-    onStderr: serverErr ? (chunk) => serverErr.push(chunk) : (chunk) => log.taskStderr(node, chunk),
+    onStdout: serverMasked ? serverMasked.out : ends.onStdout!,
+    onStderr: serverMasked ? serverMasked.err : ends.onStderr!,
     ...(args.liveChildren !== undefined ? { liveChildren: args.liveChildren } : {}),
     ...(args.track !== undefined
       ? { onSpawn: (pid: number) => void args.track!(node.id, pid) }
@@ -221,8 +259,11 @@ export async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutc
     // A shell that exited before ready can leave its group running
     // (`server & …; exit`): never registered, so no teardown reached it,
     // and it held its port against the next `vx watch` cycle's server.
-    if (err instanceof PersistentReadyError && err.reason === 'exited')
-      await terminateChildren(() => [spawn.child])
+    if (err instanceof PersistentReadyError && err.reason === 'exited' && spawn.child !== undefined)
+      await terminateChildren(() => [spawn.child!])
+    // What the masked streams hold back (a tail that could begin a secret)
+    // belongs above vx's own line: left to the idle flush, it came after it.
+    serverMasked?.end()
     // A server the run's stop killed while it started is aborted, as any
     // task the stop kills (item 962): it read `failed (never ready:
     // exited, exit 130)` with a recap after every Ctrl-C (C-62). The stop
@@ -243,7 +284,30 @@ export async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutc
     // The task's OWN stream, not the process's: the frame is where a
     // reader looks for why a task failed, and a run with a custom logger
     // (an embedder) never saw a bare stderr write at all.
-    log.taskStderr(node, `\n[vx] ${node.id}: persistent task failed to become ready: ${message}\n`)
+    log.taskStderr(
+      node,
+      ends.own(`[vx] ${node.id}: persistent task failed to become ready: ${message}\n`),
+    )
+    // A server's 127, 126 or signal death needs the line a one-shot task's
+    // gets (execute-task.ts): a dev server whose tool sits in a sibling's
+    // bin said only "exit 127".
+    if (
+      err instanceof PersistentReadyError &&
+      err.reason === 'exited' &&
+      err.exitCode !== undefined
+    ) {
+      const signal = spawn.child?.signalCode
+      const verdict = shellVerdict({
+        code: err.exitCode,
+        command: step.command,
+        cwd: node.projectDir,
+        bins: taskBinDirs(node, args.workspaceRoot),
+        signal: signal ?? undefined,
+        hidden: serverSandbox && ((f: string) => !sandboxReads(serverSandbox, f)),
+      })
+      if (verdict !== undefined)
+        log.taskStderr(node, `${serverSecrets?.mask(verdict) ?? verdict}\n`)
+    }
     // A server is never traced, so the sandbox's refusals reach no report:
     // a dev server that died on a file outside its grants read only as the
     // tool's own "not found" (2026-10-03). Said for a sandboxed server that
@@ -280,7 +344,7 @@ export async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutc
       // A server that traps the TERM and exits 0 still timed out, as a
       // one-shot task that does so reports 143 (X-116).
       exitCode:
-        ready?.reason === 'timeout'
+        ready?.reason === 'timeout' && spawn.child !== undefined
           ? (await spawn.child.exited) || signalExitCode('SIGTERM')
           : (ready?.exitCode ?? 1),
       durationMs: spawn.readyMs(),
@@ -304,16 +368,21 @@ export async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutc
       wallclockEndNs: process.hrtime.bigint() - args.runStartHrTimeNs,
     }
   }
-  args.persistentRegistry?.set(node.id, spawn.child)
+  // Ready, so it spawned: only a failed spawn has no child.
+  const child = spawn.child!
+  args.persistentRegistry?.set(node.id, child)
   forgetUndeclaredWrites(args, undeclaredWriteReach(node, args.workspaceRoot))
   if (mayWriteFingerprint(node, args.workspaceRoot, args.fingerprintWatch?.names ?? [])) {
     args.fingerprintWatch?.wrote()
   }
+  // Ready by its clean exit (a setup step): nothing is running any more.
+  const exited = child.exitCode
   return {
     node,
     status: 'success',
     exitCode: 0,
     durationMs: spawn.readyMs(),
+    ...(exited === null ? {} : { serverExit: exited }),
     ...(hash !== undefined ? { hash } : {}),
     wallclockStartNs,
     wallclockEndNs: process.hrtime.bigint() - args.runStartHrTimeNs,

package/src/orchestrator/persistent.ts

@@ -176,9 +176,8 @@ export function sessionEnd(keepAlive: KeepAlive): Promise<{ code: number; i: num
 
 export interface HoldArgs {
   forwardArgs: readonly string[] | undefined
-  /** Where a held server's output goes. */
+  /** Where a held server's output, and the notice of its death, go. */
   outOf: (id: string) => HeldServer['out']
-  warn: (message: string) => void
 }
 
 /** The kept servers handed to the caller (`RunOptions.hold`) instead of waited on. */
@@ -193,17 +192,24 @@ export function holdServers(keepAlive: KeepAlive, args: HoldArgs): HeldPersisten
   const servers = new Map<string, HeldServer>()
   keepAlive.nodes.forEach((n, i) => {
     const child = held[i]!
+    const out = args.outOf(n.id)
     servers.set(n.id, {
       child,
       config: serverConfig(n, args.forwardArgs),
-      out: args.outOf(n.id),
+      out,
       handOver: () => void handed.add(child),
     })
     // One that died during the graph was said then.
     if (hasEnded(child)) return
+    // Said where its output goes: a later run that keeps it owns the
+    // terminal from its start, and this run's renderer wrote over its rows.
     void child.exited.then((code) => {
       if (!stopping && !handed.has(child) && code !== 0)
-        args.warn(`vx: ${n.id} exited with code ${code}`)
+        out.bus.emit({
+          kind: 'run:status',
+          line: `vx: ${n.id} exited with code ${code}`,
+          warn: true,
+        })
     })
   })
   return {
@@ -260,6 +266,12 @@ export async function disposeServers(args: {
   endedBeforeStop: ReadonlySet<Child> | undefined
 }): Promise<{ keepAlive: KeepAlive; crashed: CrashedPersistent[] }> {
   const { registry, outcomes, endedBeforeStop } = args
+  // One turn of the event loop first, so Bun reaps what has already died:
+  // `exitCode` stays null until it does. A server dead at once, with
+  // nothing between its spawn and here to yield, was judged still up: the
+  // footer counted it a success, pinned it `running`, and a teardown read
+  // the run ok over vx's exit 1.
+  if (registry.size > 0) await new Promise((resolve) => setTimeout(resolve, 0))
   let keepAlive = selectKeepAlive(registry, args.nodes, args.keep)
   const failedElsewhere =
     ![...outcomes.values()].every((o) => isPassStatus(o.status)) ||