@vzn/vx 0.0.638 → 0.0.640

package/src/cache/linked-inputs.ts

@@ -0,0 +1,87 @@
+// A symlinked input keys the content behind it, as Nx reads a linked file
+// where it points (owner, 2026-10-16: "Do like nx"). Its own fold stays the
+// blob of its target STRING (file-hashes.ts), so a retarget to equal bytes
+// moves the key; this is what follows it. The target string alone let an
+// edit behind the link replay the old output (Growth, 2026-10-16), wherever
+// the target lies: in the project, elsewhere in the workspace, outside it.
+// A directory link keys every file beneath it, tracked or not, where Nx
+// does not enter one: an edit there is an edit to what the task reads.
+// `--affected` (workspace/affected.ts) and `vx watch` (cli/watch.ts) read
+// the same targets.
+
+import { readdirSync, realpathSync, statSync } from 'node:fs'
+import path from 'node:path'
+import { xxh3hex } from '../util/index.js'
+import { absentOr } from './key-fold.js'
+
+/** What a dangling link, or a loop of links, holds behind it: nothing to read. */
+const NOTHING_BEHIND = ''
+
+/**
+ * Directories a linked directory's walk never enters, as Nx's walker never
+ * does (its HARDCODED_IGNORE_PATTERNS): a link to a workspace would
+ * otherwise key the cache each run writes.
+ */
+const SKIPPED = ['.git', 'node_modules', '.vx/cache', '.yarn/cache']
+const skipped = (rel: string): boolean => SKIPPED.some((s) => rel === s || rel.endsWith(`/${s}`))
+
+/**
+ * The files under `dir` a linked directory keys, relative and `/`-separated,
+ * sorted: every regular file and link at any depth. A link is listed, never
+ * entered (`behindLink` follows it), so a loop of directories ends.
+ */
+export function filesBehind(dir: string): string[] {
+  const out: string[] = []
+  const walk = (abs: string, rel: string): void => {
+    let entries
+    try {
+      entries = readdirSync(abs, { withFileTypes: true })
+    } catch {
+      return
+    }
+    for (const e of entries) {
+      const childRel = rel === '' ? e.name : `${rel}/${e.name}`
+      if (e.isDirectory()) {
+        if (!skipped(childRel)) walk(path.join(abs, e.name), childRel)
+      } else if (e.isFile() || e.isSymbolicLink()) out.push(childRel)
+    }
+  }
+  walk(dir, '')
+  return out.sort()
+}
+
+/**
+ * The content behind the symlink at `link`, as one string: a file's identity
+ * (`hashFile` of its real path), a directory's files and theirs, or
+ * `NOTHING_BEHIND` for a dangling link or a loop. A special file (a FIFO, a
+ * device) is named, never read: a read can block. Never memoized: an upstream
+ * task may write behind the link between two keys.
+ */
+export async function behindLink(
+  link: string,
+  hashFile: (file: string) => Promise<string>,
+  seen: ReadonlySet<string> = new Set(),
+): Promise<string> {
+  let real: string
+  let st
+  try {
+    real = realpathSync(link)
+    st = statSync(real)
+  } catch {
+    return NOTHING_BEHIND
+  }
+  if (seen.has(real)) return NOTHING_BEHIND
+  if (st.isFile()) return `file\0${await hashFile(real).catch(absentOr)}`
+  if (!st.isDirectory()) return 'special'
+  const inner = new Set(seen).add(real)
+  const parts = await Promise.all(
+    filesBehind(real).map(async (rel) => {
+      const abs = path.join(real, rel)
+      const id = await hashFile(abs).catch(absentOr)
+      return id.startsWith('120000:')
+        ? `${rel}\0${id}\0${await behindLink(abs, hashFile, inner)}`
+        : `${rel}\0${id}`
+    }),
+  )
+  return `dir\0${xxh3hex(parts.join('\n'))}`
+}

package/src/cache/plugin-cache.ts

@@ -9,7 +9,7 @@
 import type { Database } from 'bun:sqlite'
 import { xxh3hex } from '../util/index.js'
 import type { CachePolicy } from './policy.js'
-import type { RemoteCacheLayer } from './layered-cache.js'
+import type { ResolvedRemote } from './remote-lane.js'
 import { scopedKey } from './scoped-cache.js'
 import { runReadKeys } from './run-store.js'
 
@@ -37,18 +37,11 @@ export interface PluginCache {
   getRun(name: string): Promise<Uint8Array | null>
 }
 
-/** One remote as the plugin cache reaches it: its wire and what the policy lets it do. */
-export interface PluginCacheRemote {
-  readonly remote: RemoteCacheLayer
-  readonly read: boolean
-  readonly write: boolean
-}
-
 export interface PluginCacheHost {
   /** The run's index; its store holds `plugin_blobs`. */
   readonly db: Database
   readonly policy: CachePolicy
-  readonly remotes: readonly PluginCacheRemote[]
+  readonly remotes: readonly ResolvedRemote[]
   readonly scope: string | undefined
 }
 
@@ -174,7 +167,6 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
       if (policy.localWrite) localPut(key, bytes)
       for (const { remote } of writers) {
         try {
-          if (await remote.has(key)) continue
           await remote.put(key, new Blob([bytes]), { durationMs: 0 })
         } catch {
           // the other layers still keep it

package/src/cache/remote-lane.ts

@@ -0,0 +1,383 @@
+// One declared remote as `LayeredCache` reaches it: what the run lets it
+// do, the boundary its answers cross (a plugin's shape is checked, its throw
+// becomes a miss), its own upload pool, and its failures said once per
+// class with the endpoint named. The order across remotes, the local store
+// and provenance are `LayeredCache`'s; nothing here knows another remote.
+
+import type { RemoteKeyTraffic, RemoteTraffic } from './cache.js'
+import type { RemoteCacheLayer } from './layered-cache.js'
+
+/** One remote, resolved: its wire and what the run's policy and its `access` let it do. */
+export interface ResolvedRemote {
+  readonly remote: RemoteCacheLayer
+  readonly read: boolean
+  readonly write: boolean
+  /** Told of each failure this remote degrades to a miss; the default writes stderr. */
+  readonly onError?: (err: Error) => void
+}
+
+/**
+ * Cap on concurrent background PUTs per remote. Keeps a burst of cache
+ * misses from opening one socket per task; excess uploads queue and drain
+ * FIFO.
+ */
+const UPLOAD_CONCURRENCY = 4
+
+/** The seam call a failure happened in, as the warning names it. */
+export type RemoteOp = 'probe' | 'download' | 'upload'
+
+const PREPOSITION: Record<RemoteOp, string> = { probe: 'at', download: 'from', upload: 'to' }
+
+/**
+ * What a remote layer resolves is a plugin's, so its shape is a boundary
+ * (item 252): a `get` that resolved `{ body: 'abc' }` was reported as
+ * "corrupt artifact … not a readable archive" — the bytes blamed for the
+ * plugin's shape — and a `hasMany` that resolved an array reached the
+ * prefetch pass's `.has()`. Named here and degraded to a miss, like every
+ * other remote failure.
+ */
+function invalidRemoteResult(what: string): Error {
+  return new Error(
+    `remote cache layer returned an invalid result: ${what} — a plugin bug, degraded to a miss`,
+  )
+}
+
+function isBody(value: unknown): value is Blob | Response {
+  return value instanceof Blob || value instanceof Response
+}
+
+function describeValue(value: unknown): string {
+  if (value === null) return 'null'
+  if (Array.isArray(value)) return 'an array'
+  // The pre-stream contract's shape, named so an unported layer reads its fix.
+  if (value instanceof ArrayBuffer || ArrayBuffer.isView(value))
+    return `a ${value.constructor.name}`
+  return typeof value === 'object' ? 'an object' : typeof value
+}
+
+/** The layer's `endpoint` as it may be printed: no userinfo, query or fragment. */
+function printableEndpoint(endpoint: unknown): string | undefined {
+  if (typeof endpoint !== 'string' || endpoint === '') return undefined
+  if (!URL.canParse(endpoint)) return endpoint
+  const u = new URL(endpoint)
+  if (u.username === '' && u.password === '' && u.search === '' && u.hash === '') return endpoint
+  u.username = ''
+  u.password = ''
+  u.search = ''
+  u.hash = ''
+  return u.toString()
+}
+
+/**
+ * What makes two failures "the same way": the error's `code` when it
+ * carries one (a gRPC status, an errno, Bun's `ConnectionRefused`), because
+ * gRPC writes the elapsed time into its message and no two deadline messages
+ * match; else the message with the artifact's hash taken out, so a 500 on
+ * one hash and a 500 on the next are one class. The operation is not part of
+ * it: an unreachable server fails the probe, the download and the upload
+ * alike, and that is one fact.
+ */
+function failureClass(err: Error, hash: string | undefined): string {
+  const code = (err as { code?: unknown }).code
+  if (typeof code === 'string' || typeof code === 'number') return `${err.name}\0${code}`
+  return hash === undefined ? err.message : err.message.replaceAll(hash, '\0')
+}
+
+/** A batch probe names no one hash, so its line counts them. */
+function batchOf(hashes: readonly string[]): { batch: string } {
+  return { batch: `of ${hashes.length} artifact${hashes.length === 1 ? '' : 's'}` }
+}
+
+export class RemoteLane {
+  readonly read: boolean
+  readonly write: boolean
+  private readonly remote: RemoteCacheLayer
+  private readonly endpoint: string | undefined
+  private readonly onError: ((err: Error) => void) | undefined
+
+  /**
+   * Hashes a batch probe found absent here, so a later fetch skips this
+   * remote without a round-trip. Same point-in-time staleness as the probe.
+   */
+  private readonly absent = new Set<string>()
+  /** What a `hasMany` said this remote holds, as `absent` records the rest. */
+  private readonly held = new Set<string>()
+
+  /**
+   * A remote the run writes but does not read (`--force`): which of the
+   * run's up-front keys it already holds, from one `hasMany`. `found` is
+   * null when the probe failed.
+   */
+  private presence: Promise<{ asked: Set<string>; found: Set<string> | null } | null> | undefined
+
+  private readonly uploadQueue: Array<() => Promise<void>> = []
+  private activeUploads = 0
+  private drainWaiters: Array<() => void> = []
+  /** Set by `stop()`: the run was stopped, so no upload starts and the drain waits on none. */
+  private stopped = false
+  private onStop: () => void = () => {}
+  private readonly stopping = new Promise<void>((resolve) => {
+    this.onStop = resolve
+  })
+
+  /**
+   * One entry per failure class said: its first line went out, `repeats`
+   * counts the ones held back. A run's requests are concurrent, so an
+   * unreachable server otherwise printed its bare runtime message once per
+   * request (three lines for one task over the Turbo wire, item 749).
+   */
+  private readonly reported = new Map<string, { cause: string; repeats: number }>()
+
+  constructor(
+    resolved: ResolvedRemote,
+    private readonly traffic: RemoteTraffic,
+  ) {
+    this.remote = resolved.remote
+    this.read = resolved.read
+    this.write = resolved.write
+    this.onError = resolved.onError
+    this.endpoint = printableEndpoint(resolved.remote.endpoint)
+  }
+
+  /**
+   * Existence probe; a failure is a miss. A key a `hasMany` this run
+   * answered for is not asked again: `--dry` batches its keys first.
+   */
+  async has(hash: string): Promise<boolean> {
+    if (this.held.has(hash)) return true
+    if (this.absent.has(hash)) return false
+    try {
+      return Boolean(await this.remote.has(hash))
+    } catch (err) {
+      this.report('probe', hash, err)
+      return false
+    }
+  }
+
+  /**
+   * The subset of `hashes` stored here in one round-trip, recording the
+   * rest as absent; `null` when this remote cannot batch or the probe
+   * failed, which the caller reads as "no batch info".
+   */
+  async hasMany(hashes: readonly string[]): Promise<Set<string> | null> {
+    if (this.remote.hasMany === undefined) return null
+    let found: unknown
+    try {
+      found = await this.remote.hasMany(hashes)
+    } catch (err) {
+      this.report('probe', batchOf(hashes), err)
+      return null
+    }
+    if (found === null || found === undefined) return null
+    if (!(found instanceof Set)) {
+      this.report(
+        'probe',
+        batchOf(hashes),
+        invalidRemoteResult(
+          `hasMany() resolved ${describeValue(found)} (expected a Set of the hashes present, or null)`,
+        ),
+      )
+      return null
+    }
+    const set = found as Set<string>
+    for (const h of hashes) (set.has(h) ? this.held : this.absent).add(h)
+    return set
+  }
+
+  /** The artifact's body, or null on a miss, a known absence, a failure or a malformed answer. */
+  async get(hash: string): Promise<{ body: Blob | Response; durationMs?: number } | null> {
+    if (this.absent.has(hash)) return null
+    let result: unknown
+    try {
+      result = await this.remote.get(hash)
+    } catch (err) {
+      this.report('download', hash, err)
+      return null
+    }
+    if (!result) return null
+    if (typeof result !== 'object' || !isBody((result as { body?: unknown }).body)) {
+      const shape =
+        typeof result !== 'object'
+          ? describeValue(result)
+          : `body is ${describeValue((result as { body?: unknown }).body)}`
+      this.report(
+        'download',
+        hash,
+        invalidRemoteResult(
+          `get() resolved ${shape} (expected { body: Blob | Response, durationMs } or null)`,
+        ),
+      )
+      return null
+    }
+    return result as { body: Blob | Response; durationMs?: number }
+  }
+
+  /**
+   * Start the batch probe `presence` reads. A lane that reads, never
+   * writes, or cannot batch (no `hasMany`) ignores it: each upload then
+   * asks `has` for its own key.
+   */
+  probeBeforeUpload(hashes: Promise<readonly string[]>): void {
+    if (this.read || !this.write || this.remote.hasMany === undefined) return
+    this.presence = hashes.then(
+      async (asked) =>
+        asked.length === 0 ? null : { asked: new Set(asked), found: await this.hasMany(asked) },
+      () => null,
+    )
+  }
+
+  /**
+   * Whether this remote already holds `hash`, asked only by a lane the run
+   * does not read: an entry is immutable, so a key it holds is the same
+   * bytes and is not sent again. Answered from the batch probe when it
+   * covered the key, else by one `has`; a failure reads as absent, and the
+   * upload goes as before.
+   */
+  private async holds(hash: string): Promise<boolean> {
+    const probed = await this.presence
+    if (probed !== undefined && probed !== null && probed.found !== null && probed.asked.has(hash))
+      return probed.found.has(hash)
+    return this.has(hash)
+  }
+
+  /** Queue a PUT in this remote's bounded pool; `body` is asked for when the job starts. */
+  upload(
+    hash: string,
+    durationMs: number,
+    body: () => { body: Blob; release?: () => Promise<void> },
+    tally: { count: number; bytes: number; ms: number; failed: number },
+  ): void {
+    const key = this.trafficOf(hash)
+    key.uploading++
+    this.traffic.inFlight++
+    this.uploadQueue.push(async () => {
+      let release: (() => Promise<void>) | undefined
+      try {
+        if (this.stopped) return
+        if (!this.read && (await this.holds(hash))) return
+        if (this.stopped) return
+        const start = performance.now()
+        const got = body()
+        release = got.release
+        await this.remote.put(hash, got.body, { durationMs })
+        tally.count++
+        tally.bytes += got.body.size
+        tally.ms += Math.round(performance.now() - start)
+        key.landed = true
+      } catch (err) {
+        tally.failed++
+        this.report('upload', hash, err)
+      } finally {
+        await release?.()
+        key.uploading--
+        this.traffic.inFlight--
+        // Nothing sent and nothing failed (the remote held it, or the run
+        // stopped first): no traffic, so the footer counts no upload.
+        if (key.uploading === 0 && !key.landed && key.failures.length === 0)
+          this.traffic.keys.delete(hash)
+      }
+    })
+    this.pump()
+  }
+
+  /** Resolves once every queued and in-flight upload has settled, or the run was stopped. */
+  async drain(): Promise<void> {
+    // A probe no upload waited on still runs: settled before the run tears
+    // the plugin down.
+    await Promise.race([this.presence, this.stopping])
+    if (this.stopped || (this.activeUploads === 0 && this.uploadQueue.length === 0)) return
+    await new Promise<void>((resolve) => this.drainWaiters.push(resolve))
+  }
+
+  /**
+   * The run was stopped (Ctrl-C, SIGTERM, an embedder's abort): queued
+   * uploads never start and the drain stops waiting on the ones in flight,
+   * which a plugin cannot be asked to cancel. A slow remote held a stopped
+   * run past the signal's exit bound, which then exited with no footer.
+   */
+  stop(): void {
+    this.stopped = true
+    this.onStop()
+    for (const resolve of this.drainWaiters.splice(0)) resolve()
+  }
+
+  /** Say the held-back repeats; the run drained first, so the counts are final. */
+  close(): void {
+    for (const { cause, repeats } of this.reported.values()) {
+      if (repeats === 0) continue
+      this.emit(
+        new Error(
+          `${repeats} more request${repeats === 1 ? '' : 's'} failed the same way: ${cause}`,
+        ),
+      )
+    }
+    this.reported.clear()
+  }
+
+  /**
+   * `subject` is the artifact's hash, or the batch for a `hasMany`. The
+   * wire's own message says what went wrong but not on what or where: an
+   * upload deadline read `The operation timed out.` with no upload, hash or
+   * server in it (item 749).
+   */
+  report(op: RemoteOp, subject: string | { batch: string }, err: unknown): void {
+    const e = err instanceof Error ? err : new Error(String(err))
+    const hash = typeof subject === 'string' ? subject : undefined
+    // A batch probe's failure is no one task's: each of its hashes is asked
+    // again on its own, and that ask is the one counted.
+    if (hash !== undefined) {
+      const failures = this.trafficOf(hash).failures
+      const said = `${op}: ${e.message}`
+      if (!failures.includes(said)) failures.push(said)
+    }
+    const cls = failureClass(e, hash)
+    const seen = this.reported.get(cls)
+    if (seen !== undefined) {
+      seen.repeats++
+      return
+    }
+    this.reported.set(cls, { cause: e.message, repeats: 0 })
+    const what = typeof subject === 'string' ? subject : subject.batch
+    const where = this.endpoint === undefined ? '' : ` ${PREPOSITION[op]} ${this.endpoint}`
+    this.emit(new Error(`${op} ${what}${where} failed: ${e.message}`, { cause: e }))
+  }
+
+  private trafficOf(hash: string): RemoteKeyTraffic {
+    let key = this.traffic.keys.get(hash)
+    if (key === undefined) {
+      key = { uploading: 0, landed: false, failures: [] }
+      this.traffic.keys.set(hash, key)
+    }
+    return key
+  }
+
+  private pump(): void {
+    while (this.activeUploads < UPLOAD_CONCURRENCY && this.uploadQueue.length > 0) {
+      const job = this.uploadQueue.shift()!
+      this.activeUploads++
+      // Jobs never reject (each wraps its PUT in the never-fail guard).
+      void job().finally(() => {
+        this.activeUploads--
+        this.pump()
+      })
+    }
+    if (this.activeUploads === 0 && this.uploadQueue.length === 0 && this.drainWaiters.length > 0) {
+      const waiters = this.drainWaiters
+      this.drainWaiters = []
+      for (const w of waiters) w()
+    }
+  }
+
+  private emit(e: Error): void {
+    // The remote cache is fully optional: NO remote failure — a 500, a
+    // network drop, a corrupt artifact, even a throwing reporter — may
+    // ever fail the run. A buggy reporter can't turn an optional-cache
+    // hiccup into a run failure.
+    try {
+      if (this.onError) this.onError(e)
+      else process.stderr.write(`[vx] remote cache: ${e.message}\n`)
+    } catch {
+      // swallow — reporting must never escalate
+    }
+  }
+}

package/src/cache/run-history.ts

@@ -13,11 +13,11 @@ const INSERT_RUNS = `
         started_at, ended_at,
         run_id, cpu_ms, peak_rss_bytes, wallclock_start_ns, wallclock_end_ns,
         cache_hit, attempts, cached,
-        blocked_by, timed_out, sandbox_violations, not_ready, restored
+        blocked_by, timed_out, sandbox_violations, not_ready, restored, remote_error
       )
       VALUES `
-/** One `runs` row's placeholders: the 22 columns `bindRun` fills. */
-const RUN_TUPLE = `(${Array(22).fill('?').join(', ')})`
+/** One `runs` row's placeholders: the 23 columns `bindRun` fills. */
+const RUN_TUPLE = `(${Array(23).fill('?').join(', ')})`
 
 /** An INSERT of `n` `runs` rows. */
 function insertRunsSql(n: number): string {
@@ -25,7 +25,7 @@ function insertRunsSql(n: number): string {
 }
 
 /**
- * Rows per INSERT: 40 × 22 variables stays under 999, the oldest default
+ * Rows per INSERT: 40 × 23 variables stays under 999, the oldest default
  * ceiling. A statement per row was 8.7 ms of 1,000 rows, 40 at a time 6.4
  * (2026-10-03).
  */
@@ -80,9 +80,9 @@ export class RunHistory {
         up_to_date_count, restored_local_count, restored_remote_count,
         exit_ok,
         commit_sha, branch, dirty, ci, ci_provider,
-        host, os, arch, vx_version, tags
+        host, os, arch, vx_version
       )
-      VALUES (?, ?, ?, ?, ?, ?,  ?, ?, ?,  ?, ?, ?, ?, ?,  ?, ?, ?,  ?,  ?, ?, ?, ?, ?,  ?, ?, ?, ?, ?)
+      VALUES (?, ?, ?, ?, ?, ?,  ?, ?, ?,  ?, ?, ?, ?, ?,  ?, ?, ?,  ?,  ?, ?, ?, ?, ?,  ?, ?, ?, ?)
       ON CONFLICT(run_id) DO NOTHING
     `,
     )
@@ -204,6 +204,22 @@ export class RunHistory {
       .immediate()
   }
 
+  /**
+   * The remote failures a run's uploads met after its rows were written:
+   * the uploads drain once the history is in (run.ts).
+   */
+  recordRemoteErrors(
+    runId: string,
+    errors: readonly (readonly [hash: string, text: string])[],
+  ): void {
+    const update = this.db.prepare('UPDATE runs SET remote_error = ? WHERE run_id = ? AND hash = ?')
+    this.db
+      .transaction(() => {
+        for (const [hash, text] of errors) update.run(text, runId, hash)
+      })
+      .immediate()
+  }
+
   /**
    * Retention: the runs table grows by one row per executed task per
    * invocation — a 2000-task repo accretes ~20k rows in days. 30 days
@@ -220,7 +236,7 @@ export class RunHistory {
   }
 }
 
-/** Bind a RunRecord to one `RUN_TUPLE`'s 22 positional parameters. */
+/** Bind a RunRecord to one `RUN_TUPLE`'s 23 positional parameters. */
 function bindRun(run: RunRecord, digestValue: (v: string) => string): SQLQueryBindings[] {
   return [
     // The ONE place the no-key sentinel is applied, so the column's
@@ -250,6 +266,7 @@ function bindRun(run: RunRecord, digestValue: (v: string) => string): SQLQueryBi
     run.sandboxViolations ?? null,
     run.notReady ?? null,
     run.restored === undefined ? null : run.restored ? 1 : 0,
+    run.remoteError ?? null,
   ]
 }
 
@@ -287,6 +304,5 @@ function bindInvocation(inv: InvocationRecord): SQLQueryBindings[] {
     inv.os,
     inv.arch,
     inv.vxVersion,
-    inv.tags,
   ]
 }

package/src/cache/run-store.ts

@@ -6,7 +6,7 @@
 // the cache is free). Plugins read them by name (`ctx.cache.getRun`).
 
 import { xxh3hex } from '../util/index.js'
-import type { PluginCacheRemote } from './plugin-cache.js'
+import type { ResolvedRemote } from './remote-lane.js'
 import { scopedKey } from './scoped-cache.js'
 import { tarPackInto, tarSize } from './tar-stream.js'
 
@@ -24,7 +24,7 @@ export function runReadKeys(name: string, scope: string | undefined): string[] {
 }
 
 /** Whether some remote takes this run's writes: only then does core keep run bytes. */
-export function writesRunStore(remotes: readonly PluginCacheRemote[]): boolean {
+export function writesRunStore(remotes: readonly ResolvedRemote[]): boolean {
   return remotes.some((r) => r.write)
 }
 
@@ -34,7 +34,7 @@ export function writesRunStore(remotes: readonly PluginCacheRemote[]): boolean {
  * a run keeps here is a record, never its verdict.
  */
 export async function putRunEntry(
-  remotes: readonly PluginCacheRemote[],
+  remotes: readonly ResolvedRemote[],
   scope: string | undefined,
   name: string,
   bytes: Uint8Array,
@@ -44,7 +44,7 @@ export async function putRunEntry(
   for (const { remote, write } of remotes) {
     if (!write) continue
     try {
-      if (!(await remote.has(key))) await remote.put(key, new Blob([bytes]), { durationMs: 0 })
+      await remote.put(key, new Blob([bytes]), { durationMs: 0 })
       kept = true
     } catch {
       // the other remotes still keep it

package/src/cache/schema.ts

@@ -98,7 +98,10 @@ export function createTables(db: Database, store: 'main' | 'store' = 'main'): vo
       not_ready           TEXT,
       -- v31: on a hit, 1 when outputs were restored, 0 when the disk
       -- already matched (up-to-date); NULL on every other status.
-      restored            INTEGER
+      restored            INTEGER,
+      -- v40: each remote read or write of the key that failed, as
+      -- 'upload: <message>', '; '-joined; NULL when none did.
+      remote_error        TEXT
     );
     -- Two whole-table indexes only, and both APPEND: every row of a run carries the
     -- same run_id and a started_at newer than everything before it, so
@@ -199,7 +202,7 @@ export function createTables(db: Database, store: 'main' | 'store' = 'main'): vo
     );
     -- v22 (Tier 3): one header row per vx-run invocation. The runs
     -- table is per-task; this is the per-invocation record that
-    -- carries git/CI/host context, the command, tags, and run-level
+    -- carries git/CI/host context, the command, and run-level
     -- counts so the dashboard never reconstructs a header with a
     -- lossy GROUP BY over runs.
     CREATE TABLE IF NOT EXISTS main.invocations (
@@ -230,8 +233,7 @@ export function createTables(db: Database, store: 'main' | 'store' = 'main'): vo
       host              TEXT,
       os                TEXT,
       arch              TEXT,
-      vx_version        TEXT NOT NULL,
-      tags              TEXT NOT NULL DEFAULT '{}'
+      vx_version        TEXT NOT NULL
     );
     CREATE INDEX IF NOT EXISTS main.invocations_started ON invocations(started_at);
     CREATE INDEX IF NOT EXISTS main.invocations_branch  ON invocations(branch);

package/src/cache/scoped-cache.ts

@@ -33,7 +33,10 @@ export class ScopedCache implements CacheLayer {
   readonly local?: CacheLayer['local']
   readonly hasRemote?: boolean
   readonly drainUploads?: () => Promise<void>
+  readonly stopUploads?: () => void
   readonly remoteHasMany?: (hashes: readonly string[]) => Promise<Set<string> | null>
+  /** A scoped run saves under the scope's key, so that is the key asked. */
+  readonly probeBeforeUpload?: (hashes: Promise<readonly string[]>) => void
   readonly getMany?: (
     hashes: readonly string[],
     ctx?: (hash: string) => CacheGetContext,
@@ -48,7 +51,11 @@ export class ScopedCache implements CacheLayer {
     if (inner.local !== undefined) this.local = inner.local
     if (inner.hasRemote !== undefined) this.hasRemote = inner.hasRemote
     if (inner.drainUploads !== undefined) this.drainUploads = () => inner.drainUploads!()
+    if (inner.stopUploads !== undefined) this.stopUploads = () => inner.stopUploads!()
     if (inner.remoteHasMany !== undefined) this.remoteHasMany = (h) => this.scopedHasMany(h)
+    if (inner.probeBeforeUpload !== undefined)
+      this.probeBeforeUpload = (h) =>
+        inner.probeBeforeUpload!(h.then((keys) => keys.map((k) => this.own(k))))
     if (inner.getMany !== undefined) this.getMany = (h, ctx) => this.scopedGetMany(h, ctx)
   }