@vzn/vx 0.0.651 → 0.0.653

package/src/orchestrator/task-hash.ts

@@ -1,6 +1,6 @@
 import { lstatSync } from 'node:fs'
 import path from 'node:path'
-import type { TaskConfig } from '../config.js'
+import type { InputStability, TaskConfig } from '../config.js'
 import type { FoldMemo, ProjectFilesCache, WorkspaceFilesCache } from '../cache/index.js'
 import {
   ABSENT_INPUT,
@@ -9,19 +9,27 @@ import {
   behindLink,
   filesToken,
   foldHead,
+  hashFileSeen,
   type InputListing,
   type CacheKeyInput,
   type CacheLayer,
   FILE_HASH_RACY_MS,
   racyWindowMs,
   resolveInputs,
+  stampGrainMs,
   type ResolveInputsArgs,
   type GitFilesCache,
   type TaskInputComponent,
   WORKSPACE_OUTPUT_PREFIX,
 } from '../cache/index.js'
 import type { InputFile, TaskInputs } from '../exec/index.js'
-import { isGroupTask, type TaskNode, type TaskOutcome } from '../graph/index.js'
+import {
+  isGroupTask,
+  KeyFailed,
+  standInsOf,
+  type TaskNode,
+  type TaskOutcome,
+} from '../graph/index.js'
 import { GROUP_VIA } from '../history/index.js'
 import {
   isUserError,
@@ -32,6 +40,7 @@ import {
   UserError,
   xxh3hex,
 } from '../util/index.js'
+import { outputsLeft } from './run-report.js'
 import { expandGroupUpstream, filterUpstreamHashes } from './upstream.js'
 
 /**
@@ -63,8 +72,41 @@ export interface HashCache {
   graph?: string | undefined
 }
 
-export function createHashCache(folds?: FoldMemo): HashCache {
-  return {
+/**
+ * `.vx/workspace`'s `outputs.clean`, after the config stage, by the run's
+ * hash cache: a task without its own runs under it, so its key folds it
+ * (`hashableConfig`). Beside `HashCache`, not on it: that type is the
+ * package's surface.
+ */
+const workspaceClean = new WeakMap<HashCache, boolean>()
+
+/** The workspace's `outputs.clean` the run's keys were derived under. */
+export function workspaceCleanOf(hashCache: HashCache): boolean | undefined {
+  return workspaceClean.get(hashCache)
+}
+
+/**
+ * The `inputs.stability.strategy` of `VX_INPUTS_STABILITY_STRATEGY` (`run`,
+ * beating a task's own) and of `.vx/workspace` (beaten by it), by the run's
+ * hash cache: it decides which saves are kept, so a key folds the one a
+ * task runs under (`hashableConfig`). Beside `HashCache` as the clean is.
+ */
+const inputStrategies = new WeakMap<HashCache, InputStrategies>()
+
+type InputStrategy = InputStability['strategy']
+type InputStrategies = { run?: InputStrategy; workspace?: InputStrategy }
+
+/** The run-level `inputs.stability.strategy` values the run's keys were derived under. */
+export function inputStrategiesOf(hashCache: HashCache): InputStrategies | undefined {
+  return inputStrategies.get(hashCache)
+}
+
+export function createHashCache(
+  folds?: FoldMemo,
+  cleanOutputs?: boolean,
+  strategies?: InputStrategies,
+): HashCache {
+  const hashCache: HashCache = {
     taskConfig: new WeakMap(),
     runtime: new Map(),
     workspaceRuntime: new Map(),
@@ -72,6 +114,10 @@ export function createHashCache(folds?: FoldMemo): HashCache {
     projectFiles: new Map(),
     folds,
   }
+  if (cleanOutputs !== undefined) workspaceClean.set(hashCache, cleanOutputs)
+  if (strategies?.run !== undefined || strategies?.workspace !== undefined)
+    inputStrategies.set(hashCache, strategies)
+  return hashCache
 }
 
 export interface ComputeHashArgs {
@@ -99,6 +145,15 @@ export interface ComputeHashArgs {
   probesAfterWrites?: ReadonlySet<string>
 }
 
+/** A key derivation whose throw is the task's `KeyFailed`: its command never ran. */
+export async function keyOf(derived: Promise<string>): Promise<string> {
+  try {
+    return await derived
+  } catch (err) {
+    throw new KeyFailed(err)
+  }
+}
+
 /**
  * Resolve every input the cache key depends on (file content, env
  * values, upstream hashes, config bytes, forwardArgs) and ask the cache layer to combine them. Used by both
@@ -174,6 +229,12 @@ export interface InputFact {
   path: string
   digest: string
   since: number
+  /**
+   * The ctime of the stat its digest came from, for a file hashed at the
+   * describe (an index OID was not stat'ed): a ctime still equal to it is
+   * not a write since, though the command's start shares its coarse tick.
+   */
+  ctimeMs?: number
   /**
    * For a link, the content behind it (`behindLink`), which the key folds
    * after the link's own target string. An edit there leaves the link
@@ -196,14 +257,21 @@ export async function describeTaskInputs(args: ComputeHashArgs): Promise<{
   inputs: TaskInputs
   facts: InputFact[]
   describedAt: number
+  /** Lists `inputs.upstream` with a non-cacheable upstream's outputs on disk; before any read of it. */
+  readUpstream: () => Promise<void>
   /** An input file the key did not fold that exists now (see `addedInput`). */
   added: () => string | undefined
 }> {
   const hashedAt = Date.now()
   const { input, listings } = await resolveKeyInput(args)
   const sorted = [...input.inputFiles].sort()
+  const ctimes = new Map<string, number>()
   const digests = await Promise.all(
-    sorted.map((f) => input.fileHashes?.get(f) ?? args.cache.hashFile(f).catch(absentOr)),
+    sorted.map(
+      (f) =>
+        input.fileHashes?.get(f) ??
+        hashFileSeen(args.cache, f, (c) => ctimes.set(f, c)).catch(absentOr),
+    ),
   )
   // Learned BEFORE the key folds it: an edit in between then reads as a
   // move at the re-check (a withheld save), never as content the key
@@ -229,6 +297,8 @@ export async function describeTaskInputs(args: ComputeHashArgs): Promise<{
       digest: digests[i]!,
       since: input.fileHashes?.has(f) === true ? indexedAt : hashedAt,
     }
+    const ctimeMs = ctimes.get(f)
+    if (ctimeMs !== undefined) fact.ctimeMs = ctimeMs
     const linked = behind.get(f)
     if (linked !== undefined) fact.behind = linked
     return fact
@@ -241,22 +311,44 @@ export async function describeTaskInputs(args: ComputeHashArgs): Promise<{
     a.hash < b.hash ? -1 : a.hash > b.hash ? 1 : 0,
   )
   // Output lists come from the local index — one SELECT for every upstream
-  // entry. An upstream with no entry (non-cacheable) contributes an empty list.
+  // entry. A non-cacheable upstream keeps no entry: its list is its
+  // declared outputs as they are on disk now (`readUpstream`), as it ran
+  // here and a local dependant reads them here. Left empty, a remote
+  // executor shipped the dependant without them and its result was saved
+  // under the key.
   // Read on first access: the local floor never reads it, and its every
   // miss paid a query over the whole closure (execute-task reads it up
   // front for a plugin executor).
-  const listUpstream = (): TaskInputs['upstream'] => {
+  const listUpstream = (onDisk?: ReadonlyMap<string, string[]>): TaskInputs['upstream'] => {
     const rows = args.cache.loadOutputFilesBatch(graft.map((g) => g.hash))
     return graft.map((g) => {
-      const outputs = (rows.get(g.hash) ?? []).map((r) =>
-        r.path.startsWith(WORKSPACE_OUTPUT_PREFIX)
-          ? r.path.slice(WORKSPACE_OUTPUT_PREFIX.length)
-          : relPosix(input.workspaceRoot, path.join(g.projectDir, r.path)),
-      )
+      const outputs =
+        onDisk?.get(g.taskId) ??
+        (rows.get(g.hash) ?? []).map((r) =>
+          r.path.startsWith(WORKSPACE_OUTPUT_PREFIX)
+            ? r.path.slice(WORKSPACE_OUTPUT_PREFIX.length)
+            : relPosix(input.workspaceRoot, path.join(g.projectDir, r.path)),
+        )
       return { taskId: g.taskId, hash: g.hash, outputs }
     })
   }
   let upstream: TaskInputs['upstream'] | undefined
+  const readUpstream = async (): Promise<void> => {
+    if (upstream !== undefined) return
+    const uncached = expandGroupUpstream(args.upstream).filter(
+      (u) => u.hash !== undefined && u.node.config.cache !== true,
+    )
+    // No nested project is cut out: each file found is on this disk, where
+    // a local run of the dependant reads it too.
+    const onDisk = new Map(
+      await Promise.all(
+        uncached.map(
+          async (u) => [u.node.id, await outputsLeft(u.node, input.workspaceRoot, [])] as const,
+        ),
+      ),
+    )
+    upstream ??= listUpstream(onDisk)
+  }
   // A listing from the run's enumeration is as old as it; one listed with
   // no run cache was listed during this describe.
   const listedAt = args.gitFilesCache === undefined ? hashedAt : indexedAt
@@ -264,6 +356,7 @@ export async function describeTaskInputs(args: ComputeHashArgs): Promise<{
     hash,
     facts,
     describedAt: hashedAt,
+    readUpstream,
     added: () => addedInput(listings, new Set(input.inputFiles), listedAt),
     inputs: {
       files,
@@ -287,12 +380,16 @@ export async function describeTaskInputs(args: ComputeHashArgs): Promise<{
  * a file whose ctime is older than its fact (by the racy window, as git
  * judges its index) has not been written since, and one that is not gets
  * hashed again and compared. ctime, because no writer can set it back. A
- * file that is gone has moved. One written at or after `commandFrom` (the
- * describe just before the command) has moved whatever it holds now: an
- * input changed and changed BACK while the command ran matched its digest
- * again, and the entry saved the other content's output under the key (item
- * 1015). A write before it — an upstream's — is judged by content, as before.
- * `byBytes` (`inputStability.strategy: 'mutable'`) judges a write during
+ * file that is gone has moved. One written after `commandFrom` (read off
+ * the file system's clock, `fsClockNow`, just after the describe) has moved
+ * whatever it holds now: an input changed and changed BACK while the
+ * command ran matched its digest again, and the entry saved the other
+ * content's output under the key (item 1015). A write before it — an
+ * upstream's — is judged by content, as before. A coarse clock dates a
+ * write in `commandFrom`'s own tick at it, before or after: such a ctime is
+ * the upstream's when the describe's stat saw it already
+ * (`InputFact.ctimeMs`), else the command's.
+ * `byBytes` (`inputs.stability.strategy: 'mutable'`) judges a write during
  * the command by content too, when it answers true for that file and its
  * ctime: the task says the same bytes are its own identical rewrite
  * (tsdown's package.json), not an edit undone, unless another command that
@@ -318,9 +415,14 @@ export async function movedInput(
     // Back since the key folded it gone.
     if (f.digest === ABSENT_INPUT) return f.path
     // A whole-second stamp may stand for any write in its second (two for
-    // an even one, FAT's tick), one during the command included
-    // (`racyWindowMs`).
-    if (commandFrom !== undefined && ctimeMs >= commandFrom - racyWindowMs(ctimeMs, 0)) {
+    // an even one, FAT's tick), one during the command included. No lag
+    // widens it: `commandFrom` is on the file system's clock already.
+    const grainMs = stampGrainMs(ctimeMs)
+    if (
+      commandFrom !== undefined &&
+      (ctimeMs > commandFrom ||
+        (ctimeMs >= commandFrom - grainMs && (grainMs > 0 || f.ctimeMs !== ctimeMs)))
+    ) {
       if (byBytes?.(f.path, ctimeMs) !== true) return f.path
       suspects.push(f)
       continue
@@ -340,27 +442,40 @@ export async function movedInput(
   return undefined
 }
 
-/** A task's `filesToken`, as its key's fold memo probe takes it. */
+/**
+ * A task's `filesToken`, as its key's fold memo probe takes it: the fields
+ * of `inputsArgs` the token reads, and no more. Built through `inputsArgs`,
+ * two spread objects per call were ~3 ms of 3,202 (2026-11-07).
+ */
 export function taskFilesToken(
-  args: Pick<
-    ComputeHashArgs,
-    'node' | 'workspaceRoot' | 'nestedProjectDirs' | 'gitFilesCache' | 'hashCache'
-  >,
+  args: Pick<ComputeHashArgs, 'node' | 'workspaceRoot' | 'nestedProjectDirs'> & {
+    gitFilesCache: GitFilesCache
+  },
 ): string | undefined {
-  return filesToken(inputsArgs({ ...args, upstream: [], cache: undefined as never }))
+  const node = args.node
+  const out = node.config.outputs
+  return filesToken({
+    projectDir: node.projectDir,
+    workspaceRoot: args.workspaceRoot,
+    inputs: node.config.inputs,
+    ownOutputs: out?.files ?? [],
+    ownWorkspaceOutputs: out?.workspaceFiles ?? [],
+    nestedProjectDirs: args.nestedProjectDirs,
+    gitFilesCache: args.gitFilesCache,
+  })
 }
 
 /** What `resolveInputs` is asked for a task's key. */
 function inputsArgs(args: ComputeHashArgs): ResolveInputsArgs {
   const cfg = args.node.config
-  const cacheCfg = cfg.cache
+  const cfgOut = cfg.outputs
   return {
     projectDir: args.node.projectDir,
     workspaceRoot: args.workspaceRoot,
     envSource: process.env,
     inputs: cfg.inputs,
-    ownOutputs: cacheCfg?.outputs.files ?? [],
-    ownWorkspaceOutputs: cacheCfg?.outputs.workspaceFiles ?? [],
+    ownOutputs: cfgOut?.files ?? [],
+    ownWorkspaceOutputs: cfgOut?.workspaceFiles ?? [],
     nestedProjectDirs: args.nestedProjectDirs,
     ...(args.gitFilesCache !== undefined ? { gitFilesCache: args.gitFilesCache } : {}),
     ...(args.probesAfterWrites?.has(args.node.id) === true ? { runtimeScope: args.node.id } : {}),
@@ -397,6 +512,7 @@ async function resolveKeyInput(
     cfg.inputs?.tasks,
     args.node.projectName,
     args.node.id,
+    standInsOf(args.node),
   )
   const upstreamHashes = upstreamPairs.map(([, hash]) => hash)
   // hash → upstream task id, for capture-row naming only (not folded).
@@ -487,7 +603,7 @@ export function hashTaskConfig(cfg: TaskConfig, hashCache?: HashCache): string {
   if (hashCache) {
     const cached = hashCache.taskConfig.get(cfg)
     if (cached !== undefined) return cached
-    const hash = xxh3hex(JSON.stringify(hashableConfig(cfg)))
+    const hash = xxh3hex(JSON.stringify(hashableConfig(asRun(cfg, hashCache))))
     hashCache.taskConfig.set(cfg, hash)
     return hash
   }
@@ -495,7 +611,36 @@ export function hashTaskConfig(cfg: TaskConfig, hashCache?: HashCache): string {
 }
 
 /**
- * Project the config for hashing: `exec.remote` is a pure PLACEMENT hint
+ * The config as the task runs it, where a run-level value changes what is
+ * saved: folded as the task's own field would be, and only then, so every
+ * key that never ran under one stays.
+ *
+ * A workspace `outputs.clean: false` saves whatever the outputs held before
+ * the command, a leftover included: a key that missed it hit that entry
+ * once the clean was back on. An `inputs.stability.strategy` other than
+ * `'immutable'` keeps a save the check withholds (`'unchecked'`, one whose
+ * input moved under the command): unfolded, that entry was hit once the
+ * check was back on, and a formatter it covered never ran.
+ */
+function asRun(cfg: TaskConfig, hashCache: HashCache): TaskConfig {
+  let out = cfg
+  if (workspaceClean.get(hashCache) === false && cfg.outputs && cfg.outputs.clean === undefined) {
+    out = { ...out, outputs: { ...cfg.outputs, clean: false } }
+  }
+  const strategies = inputStrategies.get(hashCache)
+  const inputs = cfg.inputs
+  if (strategies !== undefined && inputs !== undefined && cfg.exec !== undefined) {
+    const own = inputs.stability?.strategy
+    const runs = strategies.run ?? own ?? strategies.workspace ?? 'immutable'
+    if (runs !== (own ?? 'immutable')) {
+      out = { ...out, inputs: { ...inputs, stability: { ...inputs.stability, strategy: runs } } }
+    }
+  }
+  return out
+}
+
+/**
+ * Project the config for hashing: `exec.remote` (and `exec.worker`) is a pure PLACEMENT hint
  * with zero effect on outputs, so it is stripped — pinning a task to this
  * machine never busts a cache. It must not: the whole contract of a remote
  * executor is that the same command over the same inputs produces the
@@ -521,12 +666,24 @@ function hashableConfig(cfg: TaskConfig): unknown {
   // writes a config the lock never sorted (2026-10-25).
   const env = cfg.inputs?.env
   const pass = cfg.exec?.env?.passThrough
-  if (cfg.exec?.remote === undefined && sortedNames(env) && sortedNames(pass))
+  const secret = cfg.exec?.env?.secret
+  if (
+    cfg.exec?.remote === undefined &&
+    cfg.exec?.worker === undefined &&
+    sortedNames(env) &&
+    sortedNames(pass) &&
+    sortedNames(secret)
+  )
     return keysSorted(cfg)
-  const { remote: _remote, ...execRest } = cfg.exec ?? ({} as NonNullable<TaskConfig['exec']>)
+  const {
+    remote: _remote,
+    worker: _worker,
+    ...execRest
+  } = cfg.exec ?? ({} as NonNullable<TaskConfig['exec']>)
   // Env names are sets: each value is read by its name, so their order
   // keys nothing (conformance env/declared-order-irrelevant).
   if (!sortedNames(pass)) execRest.env = { ...execRest.env, passThrough: [...pass!].sort() }
+  if (!sortedNames(secret)) execRest.env = { ...execRest.env, secret: [...secret!].sort() }
   const out: TaskConfig = cfg.exec === undefined ? { ...cfg } : { ...cfg, exec: execRest }
   if (!sortedNames(env)) out.inputs = { ...cfg.inputs!, env: [...env!].sort() }
   return keysSorted(out)

package/src/orchestrator/telemetry-host.ts

@@ -12,7 +12,7 @@
 // See docs/design/observability-architecture-2026-06.md §1, §3.
 
 import type { EventBus } from '../report/index.js'
-import type { TaskNode } from '../graph/index.js'
+import type { TaskNode, TaskOutcome } from '../graph/index.js'
 import type {
   RunContextRecord,
   RunSummaryRecord,
@@ -31,6 +31,8 @@ export interface TelemetryHandle {
   emitSummary(summary: RunSummaryRecord): void
   /** Await every sink's flush (crash-isolated, time-bounded by the sink), each as a `<plugin>:telemetry` step. */
   flush(step?: <T>(id: string, work: () => Promise<T>) => Promise<T>): Promise<void>
+  /** Send the held `task.end` of each server not in `keep`; see `TelemetrySource.endServers`. */
+  endServers(final: readonly TaskOutcome[], keep?: ReadonlySet<string>): void
   /** Remove the bus subscription. Idempotent. */
   dispose(): void
   /** Sample a running task's process tree; see `TelemetrySource.track`. */
@@ -47,12 +49,17 @@ const RECORD_KINDS: Record<TelemetryRecord['kind'], true> = {
   'run.end': true,
 }
 
-/** Reject anything that is not sink-shaped, naming what arrived. */
+/**
+ * Reject anything that is not sink-shaped, naming what arrived, and return
+ * a copy of its members read once here. The sink is user input: a getter
+ * read again downstream, outside this try, threw past every guard and
+ * failed a green run.
+ */
 function checkSink(sink: unknown): TelemetrySink {
   if (typeof sink !== 'object' || sink === null) {
     throw new Error(`telemetry sink must be an object, got ${sink === null ? 'null' : typeof sink}`)
   }
-  const wants = (sink as TelemetrySink).wants
+  const { name, wants, onRecord, onRunSummary, flush } = sink as TelemetrySink
   if (wants !== undefined && !Array.isArray(wants)) {
     throw new Error(`telemetry sink 'wants' must be an array, got ${typeof wants}`)
   }
@@ -69,13 +76,24 @@ function checkSink(sink: unknown): TelemetrySink {
   // subscribed to the bus and heard nothing — the plugin was "on" and did
   // nothing. Refused here, which the caller turns into the same warning
   // a throwing hook gets: never a failed run.
-  const s = sink as TelemetrySink
-  if (typeof s.onRecord !== 'function' && typeof s.onRunSummary !== 'function') {
+  if (typeof onRecord !== 'function' && typeof onRunSummary !== 'function') {
     throw new Error(
       'telemetry sink handles nothing: neither onRecord nor onRunSummary is a function',
     )
   }
-  return s
+  for (const [key, fn] of Object.entries({ onRecord, onRunSummary, flush })) {
+    if (fn !== undefined && typeof fn !== 'function') {
+      throw new Error(`telemetry sink '${key}' must be a function, got ${typeof fn}`)
+    }
+  }
+  // Bound: a sink written as a class reads its own state through `this`.
+  return {
+    ...(typeof name === 'string' ? { name } : {}),
+    ...(wants !== undefined ? { wants: [...wants] } : {}),
+    ...(onRecord !== undefined ? { onRecord: onRecord.bind(sink) } : {}),
+    ...(onRunSummary !== undefined ? { onRunSummary: onRunSummary.bind(sink) } : {}),
+    ...(flush !== undefined ? { flush: flush.bind(sink) } : {}),
+  }
 }
 
 /**
@@ -147,6 +165,7 @@ export async function subscribeTelemetry(
   return {
     emitSummary: (summary) => source.emitSummary(summary),
     flush: (step) => source.flush(step),
+    endServers: (final, keep) => source.endServers(final, keep),
     dispose,
     ...(source.track !== undefined ? { track: source.track } : {}),
   }