@vzn/vx 0.0.25 → 0.0.27

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vzn/vx",
-  "version": "0.0.25",
+  "version": "0.0.27",
   "description": "An open, extensible monorepo task runner.",
   "type": "module",
   "exports": {
@@ -17,10 +17,10 @@
     "node": ">=18"
   },
   "optionalDependencies": {
-    "@vzn/vx-linux-x64": "0.0.25",
-    "@vzn/vx-linux-arm64": "0.0.25",
-    "@vzn/vx-darwin-x64": "0.0.25",
-    "@vzn/vx-darwin-arm64": "0.0.25"
+    "@vzn/vx-linux-x64": "0.0.27",
+    "@vzn/vx-linux-arm64": "0.0.27",
+    "@vzn/vx-darwin-x64": "0.0.27",
+    "@vzn/vx-darwin-arm64": "0.0.27"
   },
   "dependencies": {
     "@anthropic-ai/sandbox-runtime": "^0.0.76"

package/src/cache/chained-cache.ts

@@ -3,6 +3,13 @@
 // layer owns the run index (history, stats, prune) so a run is recorded
 // once. Restore goes to the layer that produced the hit — remembered per
 // hash — because an entry's artifact lives wherever it was found.
+//
+// A layer that throws in a lookup is a miss there and the walk goes on; one
+// that throws in a save is skipped and the rest still save. Unisolated, a
+// plugin layer's throw ended the walk before the local floor under it: its
+// `get` failed the task, its `save` kept the entry from every later layer
+// (item 1020). A save that fails in EVERY layer still throws, so the
+// caller's own "cache save failed" line stays the one word on it.
 
 import type {
   Cache,
@@ -21,15 +28,31 @@ import type {
   RunRecord,
 } from './cache.js'
 
+/** Told of a layer's failure the chain went past: its index, the method, the error. */
+export type LayerErrorReport = (layer: number, method: string, err: unknown) => void
+
 export class ChainedCache implements CacheLayer {
   readonly hasRemote: boolean
   private readonly hitLayer = new Map<string, CacheLayer>()
 
-  constructor(readonly layers: readonly CacheLayer[]) {
+  constructor(
+    readonly layers: readonly CacheLayer[],
+    private readonly onLayerError: LayerErrorReport = () => undefined,
+  ) {
     if (layers.length < 2) throw new Error('ChainedCache needs at least two layers')
     this.hasRemote = layers.some((l) => l.hasRemote === true)
   }
 
+  /** `layer`'s answer, or `fallback` once its failure is reported. */
+  private async ask<T>(layer: CacheLayer, method: string, call: () => Promise<T>, fallback: T) {
+    try {
+      return await call()
+    } catch (err) {
+      this.onLayerError(this.layers.indexOf(layer), method, err)
+      return fallback
+    }
+  }
+
   get local(): Cache | undefined {
     return this.layers[0]!.local
   }
@@ -49,11 +72,11 @@ export class ChainedCache implements CacheLayer {
     // saved locally, in the outcome, the summary and telemetry (item 889).
     const owner = this.hitLayer.get(hash)
     if (owner !== undefined) {
-      const entry = await owner.get(hash, ctx)
+      const entry = await this.ask(owner, 'get', () => owner.get(hash, ctx), null)
       if (entry !== null) return entry
     }
     for (const layer of this.layers) {
-      const entry = await layer.get(hash, ctx)
+      const entry = await this.ask(layer, 'get', () => layer.get(hash, ctx), null)
       if (entry !== null) {
         this.hitLayer.set(hash, layer)
         return entry
@@ -64,7 +87,7 @@ export class ChainedCache implements CacheLayer {
 
   async has(hash: string): Promise<'local' | 'remote' | null> {
     for (const layer of this.layers) {
-      const where = await layer.has(hash)
+      const where = await this.ask(layer, 'has', () => layer.has(hash), null)
       if (where !== null) {
         this.hitLayer.set(hash, layer)
         return where
@@ -75,7 +98,7 @@ export class ChainedCache implements CacheLayer {
 
   async prefetch(hash: string, ctx?: CacheGetContext): Promise<boolean> {
     for (const layer of this.layers) {
-      if (await layer.prefetch(hash, ctx)) {
+      if (await this.ask(layer, 'prefetch', () => layer.prefetch(hash, ctx), false)) {
         this.hitLayer.set(hash, layer)
         return true
       }
@@ -96,7 +119,12 @@ export class ChainedCache implements CacheLayer {
     let complete = true
     for (const layer of this.layers) {
       if (layer.hasRemote !== true) continue
-      const found = (await layer.remoteHasMany?.(hashes)) ?? null
+      const found = await this.ask(
+        layer,
+        'remoteHasMany',
+        async () => (await layer.remoteHasMany?.(hashes)) ?? null,
+        null,
+      )
       if (found === null) {
         complete = false
         continue
@@ -159,12 +187,20 @@ export class ChainedCache implements CacheLayer {
     // `skipLocalWrite` and go straight to their remote upload (which reads
     // the artifact the first layer just wrote — same handle, same path).
     const seenLocals = new Set<NonNullable<CacheLayer['local']>>()
-    for (const layer of this.layers) {
+    const failures: Array<[layer: number, err: unknown]> = []
+    for (const [i, layer] of this.layers.entries()) {
       const local = layer.local
       const skip = local !== undefined && seenLocals.has(local)
-      await layer.save(skip ? { ...args, skipLocalWrite: true } : args)
+      try {
+        await layer.save(skip ? { ...args, skipLocalWrite: true } : args)
+      } catch (err) {
+        failures.push([i, err])
+        continue
+      }
       if (local !== undefined) seenLocals.add(local)
     }
+    if (failures.length === this.layers.length) throw failures[0]![1]
+    for (const [i, err] of failures) this.onLayerError(i, 'save', err)
   }
 
   ingest(hash: string, body: Blob | Response, meta: IngestMeta): Promise<void> {

package/src/orchestrator/plugin-host.ts

@@ -326,6 +326,7 @@ export async function resolveCache(
   ctx: CacheContext,
 ): Promise<CacheLayer> {
   const layers: CacheLayer[] = []
+  const owners = new Map<CacheLayer, string>()
   for (const plugin of plugins) {
     if (plugin.cache === undefined) continue
     const layer = await safe(plugin, 'cache', () => plugin.cache!(ctx))
@@ -337,6 +338,7 @@ export async function resolveCache(
     // passed and died at its first hit inside restoreOutputs.
     assertShape(plugin, 'cache', layer, CACHE_LAYER_METHODS, 'a cache layer')
     layers.push(layer)
+    if (!owners.has(layer)) owners.set(layer, `plugin '${plugin.name}'`)
   }
   // Core's own store is the TAIL of the chain — the floor under every
   // lookup, not a plugin a workspace has to declare. A layer that WRAPS
@@ -345,7 +347,21 @@ export async function resolveCache(
   if (!layers.includes(ctx.localCache)) layers.push(ctx.localCache)
   const wrapsLocal = layers.some((l) => l !== ctx.localCache && l.local === ctx.localCache)
   const distinct = wrapsLocal ? layers.filter((l) => l !== ctx.localCache) : layers
-  return distinct.length === 1 ? distinct[0]! : new ChainedCache(distinct)
+  if (distinct.length === 1) return distinct[0]!
+  // One line per layer and method: a layer that throws on every lookup
+  // would otherwise print once per task.
+  const told = new Set<string>()
+  return new ChainedCache(distinct, (i, method, err) => {
+    const layer = distinct[i]!
+    const who =
+      layer === ctx.localCache ? 'the local cache' : (owners.get(layer) ?? 'a cache layer')
+    if (told.has(`${i}\0${method}`)) return
+    told.add(`${i}\0${method}`)
+    const m = err instanceof Error ? err.message : String(err)
+    const then =
+      method === 'save' ? 'the other layers still save' : 'a miss there; the next layer answers'
+    ctx.warn(`[vx] ${who} failed in cache ${method}: ${m}; ${then}`)
+  })
 }
 
 /**
@@ -380,9 +396,10 @@ export async function resolveExecutors(
  * End-of-run plugin lifecycle: each plugin's optional `teardown()`, in
  * declaration order. Crash-isolated — a throwing teardown is logged and
  * skipped, never propagated — and each call is time-bounded by
- * {@link teardownTimeoutMs}. Runs on the normal completion path only; the
- * finally-path disposers just unsubscribe. (Telemetry sinks flush before
- * this, in telemetry-host.ts, with the same reporting rule.)
+ * {@link teardownTimeoutMs}. Runs on every exit of a run once its plugins'
+ * factories ran, and of a plan (item 1021); the finally-path disposers
+ * just unsubscribe. (Telemetry sinks flush before this, in
+ * telemetry-host.ts, with the same reporting rule.)
  *
  * The bound is PER CALL and the calls are sequential, so the worst case
  * composes: measured at the 3s default, 1/2/3 simultaneously-hung plugins

package/src/orchestrator/plugin.ts

@@ -442,6 +442,20 @@ export interface InstallPluginsArgs {
   warn?: (message: string) => void
 }
 
+/**
+ * A plugin's `setup` threw. It names the plugin so the run can tear down
+ * every other one: their capability factories and setups ran, and the
+ * one that failed set up nothing to release.
+ */
+export class PluginSetupError extends UserError {
+  constructor(
+    readonly plugin: Plugin,
+    message: string,
+  ) {
+    super(message)
+  }
+}
+
 /**
  * Install every plugin's `setup` hook against a shared bus + context.
  * Synchronous loop; setup() promises are awaited in order so a plugin's
@@ -534,7 +548,8 @@ export async function installPlugins(args: InstallPluginsArgs): Promise<() => vo
     try {
       await plugin.setup(ctx)
     } catch (err) {
-      throw new UserError(
+      throw new PluginSetupError(
+        plugin,
         `plugin '${plugin.name}' failed to load: ${err instanceof Error ? err.message : String(err)}`,
       )
     }

package/src/orchestrator/run.ts

@@ -39,7 +39,7 @@ import { admitTasks, taintTracker } from './admission.js'
 import { excludedTaint } from './excluded-keys.js'
 import { keyUpstream } from './upstream.js'
 import { busLogger, createEventBus, terminalSubscriber, type EventBus } from './events.js'
-import { installPlugins } from './plugin.js'
+import { installPlugins, PluginSetupError } from './plugin.js'
 import { buildAdmission, resolveExecutors, teardownPlugins } from './plugin-host.js'
 import { subscribeTelemetry, type TelemetryHandle } from './telemetry-host.js'
 import { assembleRunSummary, isPassStatus } from './telemetry.js'
@@ -217,6 +217,19 @@ async function runOnBus(
 
   const prepared = await prepareRun(options, log)
   mark('plugin stages')
+  // Every exit from here tears the plugins down: their capability
+  // factories ran in prepareRun, and teardown is what releases what those
+  // opened (a client, a channel). Until item 1021 only the normal end did,
+  // so an early return, a refused setup or a throw left them held — once
+  // per cycle under `vx watch`. A plugin whose own setup threw set up
+  // nothing, and is the one left out.
+  let tornDown = false
+  const teardown = async (except?: unknown): Promise<void> => {
+    if (tornDown) return
+    tornDown = true
+    const plugins = prepared.plugins.filter((p) => p !== except)
+    await teardownPlugins(plugins, (m) => log.status(m))
+  }
   // A task whose command re-enters `vx run` in the workspace running it
   // is refused. When the inner run reaches this task again it forks a run
   // per run until the machine gives out; when it does not (`ci` shelling
@@ -226,6 +239,7 @@ async function runOnBus(
   // name the task; the check is on the root so both shapes are caught.
   const outerRoot = process.env[VX_RUN_WORKSPACE_ENV]
   if (outerRoot !== undefined && path.resolve(outerRoot) === path.resolve(prepared.workspaceRoot)) {
+    await teardown()
     prepared.cache.close()
     throw new UserError(
       `task ${process.env[VX_RUN_TASK_ENV] ?? '<unknown>'} runs \`vx run\` inside its own workspace: a nested run is invisible to the outer graph (its tasks escape the schedule, the concurrency budget and the cache key) and a loop back to this task forks without bound. Declare what it needs with dependsOn instead.`,
@@ -241,6 +255,7 @@ async function runOnBus(
     log.status(
       `No projects declare task(s): ${prepared.unresolvedTasks.join(', ')}.${didYouMean(prepared.unresolvedTasks, prepared.projects)}${await initHint(prepared)}`,
     )
+    await teardown()
     prepared.cache.close()
     return { ok: false, outcomes: [] }
   }
@@ -254,6 +269,7 @@ async function runOnBus(
         ? `No projects declare task(s): ${options.tasks.join(', ')}.${await initHint(prepared)}`
         : 'No tasks to run.'
     log.status(msg)
+    await teardown()
     prepared.cache.close()
     return { ok: false, outcomes: [] }
   }
@@ -273,9 +289,21 @@ async function runOnBus(
     })
   } catch (err) {
     disposePlugins?.()
+    await teardown(err instanceof PluginSetupError ? err.plugin : undefined)
     prepared.cache.close()
     throw err
   }
+  // Until the run's own try below, a throw leaves through here.
+  const abandoning = async <T>(step: () => T | Promise<T>): Promise<T> => {
+    try {
+      return await step()
+    } catch (err) {
+      disposePlugins?.()
+      await teardown()
+      prepared.cache.close()
+      throw err
+    }
+  }
   const {
     workspaceRoot,
     workspaceConfig,
@@ -293,24 +321,19 @@ async function runOnBus(
 
   // Resolved ONCE per run, in declaration order, the local executor last.
   // A broken factory aborts here, before any task starts.
-  let executors: readonly TaskExecutor[]
-  try {
-    executors = await resolveExecutors(prepared.plugins, {
+  const executors: readonly TaskExecutor[] = await abandoning(() =>
+    resolveExecutors(prepared.plugins, {
       workspaceRoot: prepared.workspaceRoot,
       cacheDir: prepared.cacheDir,
       warn: (m: string) => log.status(m),
       concurrency,
-    })
-  } catch (err) {
-    disposePlugins?.()
-    prepared.cache.close()
-    throw err
-  }
+    }),
+  )
   // Placement: decided ONCE per task, before scheduling, so the scheduler
   // can admit a remote-pooled task against its pool instead of a local
   // worker slot. Group tasks run nothing; persistent tasks never reach an
   // executor (local by construction) — both stay off the map.
-  const placements = placeTasks(nodes, executors)
+  const placements = await abandoning(() => placeTasks(nodes, executors))
   // A `remote: 'only'` task nobody takes succeeds WITHOUT running. That is
   // deliberate — on a machine with no remote pool the ambient state already
   // is what the task would have produced — but it must not be SILENT: a task
@@ -342,7 +365,9 @@ async function runOnBus(
   }
   // The SAME set is handed back every time, narrowed in place: a consumer
   // reading it later sees the truth rather than a stale snapshot.
-  for (const [executor, remaining] of demandOf) executor.demand!(remaining)
+  await abandoning(() => {
+    for (const [executor, remaining] of demandOf) executor.demand!(remaining)
+  })
   const narrowDemand =
     demandOf.size === 0
       ? (): void => {}
@@ -457,10 +482,12 @@ async function runOnBus(
   }
   // Taken after the early exits above (nothing they do touches a tree) and
   // before the schedule: from here on tasks clean, restore and write.
-  const releaseRunLock = await acquireRunLock(prepared.workspaceRoot, {
-    log: (m) => log.status(m),
-    signal: stopRun.signal,
-  })
+  const releaseRunLock = await abandoning(() =>
+    acquireRunLock(prepared.workspaceRoot, {
+      log: (m) => log.status(m),
+      signal: stopRun.signal,
+    }),
+  )
   try {
     // One run-id per `vx run` invocation. Every task in the resulting
     // graph carries it so analytics queries can group by invocation.
@@ -900,8 +927,8 @@ async function runOnBus(
     }
     // End-of-run plugin lifecycle: each plugin's teardown(). Crash-isolated
     // + time-bounded inside teardownPlugins, so a faulty plugin can neither
-    // fail nor hang the run. Normal completion path only — the finally
-    // below just unsubscribes.
+    // fail nor hang the run. Here on the normal path, after the drains
+    // below; every other exit takes it through `teardown` too.
     // Drain any still-in-flight background prefetches before closing the
     // cache handle — a prefetch ingesting into a closed SQLite DB would
     // throw. Tasks that resolved as local hits never awaited their
@@ -939,7 +966,7 @@ async function runOnBus(
     // Not on a stopped run: one stopped while it waited on another run's
     // lock never held it, and its prune evicted under that run (item 858).
     if (!stopRun.signal.aborted) await applyCacheRetention(prepared, log)
-    await teardownPlugins(prepared.plugins, (m) => log.status(m))
+    await teardown()
     await closeCache()
     mark('close')
     printTimings()
@@ -1019,6 +1046,8 @@ async function runOnBus(
     // released here. Idempotent; safe even if installPlugins threw.
     disposePlugins?.()
     telemetry?.dispose()
+    // No-op after the normal path's teardown; on a throw, the only one.
+    await teardown()
     // No-op after the normal path's close. On a throw this is the only
     // close there is, and it must not itself throw — that would replace
     // the run's real error with a teardown one. Background uploads are
@@ -1123,6 +1152,9 @@ export async function planRun(options: RunOptions): Promise<RunPlan> {
     mark('plan')
     return planned
   } finally {
+    // The plan called the cache and executor factories; teardown releases
+    // what they opened, as a run's end does (item 1021).
+    await teardownPlugins(prepared.plugins, (m) => log.status(m))
     prepared.cache.close()
     // The same table a run prints: a dry run is how the prepare stages
     // (discovery, config load, the git enumeration) get profiled on a