@vzn/vx 0.0.18 → 0.0.22

package/src/cache/layered-cache.ts

@@ -1,7 +1,7 @@
 // LayeredCache — composes the local cache with a remote HTTP cache.
 //
-// Read path:  try local. On miss, try remote; on remote hit, ingest
-// the artifact bytes into local so the next read is a local hit.
+// Read path:  try local. On miss, try remote; on remote hit, stream
+// the artifact body into local so the next read is a local hit.
 //
 // Write path: write to local synchronously. Upload the local artifact
 // to remote as a fire-and-forget background task; failures log a
@@ -42,10 +42,23 @@ import { FULL_CACHE_POLICY, type OutputDirRow } from './cache.js'
  * policy gating, in-flight dedup, remote provenance, and the never-fail
  * contract (implementations THROW on failure; LayeredCache degrades every
  * throw to a cache miss via `onRemoteError`). The artifact bytes are the
- * local `<hash>.tar.zst` verbatim. See
- * docs/design/native-cache-wire-2026-07.md.
+ * local `<hash>.tar.zst` verbatim. The wires live in plugin packages
+ * (`@vzn/vx-migrate`'s `turboCache()` and `nxCache()`, `@vzn/vx-reapi`); see
+ * docs/modules/layered-cache.md.
+ *
+ * Core awaits every call and bounds none: a `get` that never settles
+ * holds its task and a `put` the run's upload drain, so each request
+ * carries the layer's own deadline (every first-party layer has one; the
+ * plugin guide's example shows it).
  */
 export interface RemoteCacheLayer {
+  /**
+   * Where the layer's artifacts live, named in every degrade warning
+   * (`upload <hash> to <endpoint> failed: …`): a base URL or `host:port`.
+   * Never a credential: a URL's `user:pass@`, query and fragment are
+   * dropped before it is printed.
+   */
+  readonly endpoint?: string
   /** Existence probe (drives the plan path's `--dry` remote prediction). */
   has(hash: string): Promise<boolean>
   /**
@@ -57,11 +70,47 @@ export interface RemoteCacheLayer {
    * info; use per-hash".
    */
   hasMany?(hashes: readonly string[]): Promise<Set<string> | null>
-  /** Fetch an artifact's bytes; `null` = miss. `durationMs` is the
-   *  producing task's duration when the wire carries it. */
-  get(hash: string): Promise<{ body: ArrayBuffer; durationMs: number | undefined } | null>
-  /** Store an artifact (fire-and-forget from LayeredCache's perspective). */
-  put(hash: string, body: ArrayBuffer | Uint8Array, meta: { durationMs: number }): Promise<void>
+  /**
+   * Fetch an artifact; `null` = miss. `body` is read once, by core,
+   * straight into the local cache — an HTTP wire returns its `fetch`
+   * `Response`, a chunked wire `new Response(stream)`, bytes in hand
+   * `new Blob([bytes])`. `durationMs` is the producing task's duration
+   * when the wire carries it.
+   */
+  get(hash: string): Promise<{ body: Blob | Response; durationMs: number | undefined } | null>
+  /**
+   * Store an artifact (fire-and-forget from LayeredCache's perspective).
+   * `body` is file-backed (`Bun.file`) when the local store holds the
+   * artifact, so a plugin that streams it never holds it whole.
+   */
+  put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void>
+}
+
+/**
+ * What a remote layer resolves is a plugin's, so its shape is a boundary
+ * (item 252): a `get` that resolved `{ body: 'abc' }` was reported as
+ * "corrupt artifact … not a readable archive" — the bytes blamed for the
+ * plugin's shape — and a `hasMany` that resolved an array reached the
+ * prefetch pass's `.has()`. Named here and degraded to a miss, like every
+ * other remote failure.
+ */
+function invalidRemoteResult(what: string): Error {
+  return new Error(
+    `remote cache layer returned an invalid result: ${what} — a plugin bug, degraded to a miss`,
+  )
+}
+
+function isBody(value: unknown): value is Blob | Response {
+  return value instanceof Blob || value instanceof Response
+}
+
+function describeValue(value: unknown): string {
+  if (value === null) return 'null'
+  if (Array.isArray(value)) return 'an array'
+  // The pre-stream contract's shape, named so an unported layer reads its fix.
+  if (value instanceof ArrayBuffer || ArrayBuffer.isView(value))
+    return `a ${value.constructor.name}`
+  return typeof value === 'object' ? 'an object' : typeof value
 }
 
 /**
@@ -70,8 +119,51 @@ export interface RemoteCacheLayer {
  */
 const UPLOAD_CONCURRENCY = 4
 
+/** The seam call a failure happened in, as the warning names it. */
+type RemoteOp = 'probe' | 'download' | 'upload'
+
+const PREPOSITION: Record<RemoteOp, string> = { probe: 'at', download: 'from', upload: 'to' }
+
+/** The layer's `endpoint` as it may be printed: no userinfo, query or fragment. */
+function printableEndpoint(endpoint: unknown): string | undefined {
+  if (typeof endpoint !== 'string' || endpoint === '') return undefined
+  if (!URL.canParse(endpoint)) return endpoint
+  const u = new URL(endpoint)
+  if (u.username === '' && u.password === '' && u.search === '' && u.hash === '') return endpoint
+  u.username = ''
+  u.password = ''
+  u.search = ''
+  u.hash = ''
+  return u.toString()
+}
+
+/**
+ * What makes two failures "the same way": the error's `code` when it
+ * carries one (a gRPC status, an errno, Bun's `ConnectionRefused`), because
+ * gRPC writes the elapsed time into its message and no two deadline messages
+ * match; else the message with the artifact's hash taken out, so a 500 on
+ * one hash and a 500 on the next are one class. The operation is not part of
+ * it: an unreachable server fails the probe, the download and the upload
+ * alike, and that is one fact.
+ */
+function failureClass(err: Error, hash: string | undefined): string {
+  const code = (err as { code?: unknown }).code
+  if (typeof code === 'string' || typeof code === 'number') return `${err.name}\0${code}`
+  return hash === undefined ? err.message : err.message.replaceAll(hash, '\0')
+}
+
+/** A batch probe names no one hash, so its line counts them. */
+function batchOf(hashes: readonly string[]): { batch: string } {
+  return { batch: `of ${hashes.length} artifact${hashes.length === 1 ? '' : 's'}` }
+}
+
 export interface LayeredCacheOptions {
-  /** Called for remote-related errors that the layer suppresses. */
+  /**
+   * Called for the remote failures the layer degrades to a miss, once per
+   * failure class per layer (`upload <hash> to <endpoint> failed: <cause>`,
+   * with the original error as `cause`); the repeats are counted and said
+   * once more at `close()` (`<n> more requests failed the same way: <cause>`).
+   */
   onRemoteError?: (err: Error) => void
   /**
    * The 4-axis read/write policy. The local slice (read/write) is
@@ -120,6 +212,15 @@ export class LayeredCache implements CacheLayer {
   private drainWaiters: Array<() => void> = []
 
   private readonly policy: CachePolicy
+  private readonly endpoint: string | undefined
+
+  /**
+   * One entry per failure class said: its first line went out, `repeats`
+   * counts the ones held back. A run's requests are concurrent, so an
+   * unreachable server otherwise printed its bare runtime message once per
+   * request (three lines for one task over the Turbo wire, item 749).
+   */
+  private readonly reported = new Map<string, { cause: string; repeats: number }>()
 
   constructor(
     readonly local: Cache,
@@ -127,10 +228,11 @@ export class LayeredCache implements CacheLayer {
     private readonly options: LayeredCacheOptions = {},
   ) {
     this.policy = options.policy ?? FULL_CACHE_POLICY
+    this.endpoint = printableEndpoint(remote.endpoint)
   }
 
-  async key(input: CacheKeyInput): Promise<string> {
-    return await this.local.key(input)
+  key(input: CacheKeyInput): Promise<string> {
+    return this.local.key(input)
   }
 
   async prefetch(hash: string, ctx?: CacheGetContext): Promise<boolean> {
@@ -140,7 +242,7 @@ export class LayeredCache implements CacheLayer {
     // point): pullFromRemote registers the `inflight` entry SYNCHRONOUSLY,
     // so a concurrent markRemoteAbsent can't clobber a pending pull — a
     // guard done here (behind an async local.has) would reopen that race.
-    return await this.pullFromRemote(hash, ctx)
+    return this.pullFromRemote(hash, ctx)
   }
 
   /**
@@ -154,9 +256,22 @@ export class LayeredCache implements CacheLayer {
   async remoteHasMany(hashes: readonly string[]): Promise<Set<string> | null> {
     if (!this.policy.remoteRead || this.remote.hasMany === undefined) return null
     try {
-      return await this.remote.hasMany(hashes)
+      // `return await`, deliberately: the catch below is the never-fail
+      // contract, and a returned promise's rejection would sail past it.
+      const found: unknown = await this.remote.hasMany(hashes)
+      if (found !== null && found !== undefined && !(found instanceof Set)) {
+        this.reportRemoteError(
+          'probe',
+          batchOf(hashes),
+          invalidRemoteResult(
+            `hasMany() resolved ${describeValue(found)} (expected a Set of the hashes present, or null)`,
+          ),
+        )
+        return null
+      }
+      return found ?? null
     } catch (err) {
-      this.reportRemoteError(err)
+      this.reportRemoteError('probe', batchOf(hashes), err)
       return null
     }
   }
@@ -224,7 +339,7 @@ export class LayeredCache implements CacheLayer {
     try {
       return (await this.remote.has(hash)) ? 'remote' : null
     } catch (err) {
-      this.reportRemoteError(err)
+      this.reportRemoteError('probe', hash, err)
       return null
     }
   }
@@ -256,16 +371,31 @@ export class LayeredCache implements CacheLayer {
     // or remote if a concurrent prefetch set it).
     if ((await this.local.has(hash)) === 'local') return true
 
-    let remoteResult
+    let remoteResult: unknown
     try {
       remoteResult = await this.remote.get(hash)
     } catch (err) {
-      this.reportRemoteError(err)
+      this.reportRemoteError('download', hash, err)
       return false
     }
     if (!remoteResult) return false
+    if (typeof remoteResult !== 'object' || !isBody((remoteResult as { body?: unknown }).body)) {
+      const shape =
+        typeof remoteResult !== 'object'
+          ? describeValue(remoteResult)
+          : `body is ${describeValue((remoteResult as { body?: unknown }).body)}`
+      this.reportRemoteError(
+        'download',
+        hash,
+        invalidRemoteResult(
+          `get() resolved ${shape} (expected { body: Blob | Response, durationMs } or null)`,
+        ),
+      )
+      return false
+    }
+    const remoteBody = remoteResult as { body: Blob | Response; durationMs?: number }
 
-    // Ingest the remote bytes into local using the caller-supplied
+    // Ingest the remote body into local using the caller-supplied
     // taskId/command plus the remote-reported durationMs. The remote
     // layer carries durationMs as an HTTP header (x-artifact-duration);
     // taskId + command come from the orchestrator's TaskNode in scope.
@@ -275,16 +405,21 @@ export class LayeredCache implements CacheLayer {
     const meta: IngestMeta = {
       taskId: ctx?.taskId ?? `${hash}#unknown`,
       command: ctx?.command ?? '',
-      durationMs: remoteResult.durationMs ?? 0,
+      // A layer's number is taken only as a finite non-negative one: a NaN
+      // failed the entry row's NOT NULL after the artifact was renamed into
+      // place, which threw a valid hit away and orphaned its bytes, and a
+      // negative or infinite one was stored and replayed (item 944).
+      durationMs: durationOf(remoteBody.durationMs),
+      ...(ctx?.outputs !== undefined ? { outputs: ctx.outputs } : {}),
     }
     try {
-      await this.local.ingest(hash, new Uint8Array(remoteResult.body), meta)
+      await this.local.ingest(hash, remoteBody.body, meta)
     } catch (err) {
       // The bytes came off the network — a corrupt/truncated remote
       // artifact must degrade to a cache miss (task re-executes), not
       // crash the run. Local-layer reads outside this block still
       // propagate: local corruption is a real fault, not a network one.
-      this.reportRemoteError(err)
+      this.reportRemoteError('download', hash, err)
       return false
     }
     this.remoteSourced.add(hash)
@@ -295,8 +430,8 @@ export class LayeredCache implements CacheLayer {
     return this.local.outputsPath(hash)
   }
 
-  async hashFile(filePath: string): Promise<string> {
-    return await this.local.hashFile(filePath)
+  hashFile(filePath: string): Promise<string> {
+    return this.local.hashFile(filePath)
   }
 
   async restoreOutputs(hash: string, projectDir: string, workspaceRoot?: string): Promise<void> {
@@ -315,16 +450,13 @@ export class LayeredCache implements CacheLayer {
     // cache. Errors are logged, not propagated: the task already
     // succeeded; we don't fail it on cache-server issues.
     //
-    // The artifact bytes are read INSIDE the job, not here.
-    // UPLOAD_CONCURRENCY bounds sockets, not memory: a queued closure
-    // holding its own artifact keeps the WHOLE backlog resident, so peak
-    // RSS scaled with a run's total miss artifact bytes rather than with
-    // the pool — any cold monorepo run whose remote uploads slower than the
-    // build produces artifacts held every pending one at once. Reading in
-    // the job caps resident artifact bytes at UPLOAD_CONCURRENCY. The
+    // The job hands the plugin a file-backed Blob over the local artifact,
+    // opened when the plugin reads it: a queued job holds a path, not a
+    // buffer, and a plugin that streams the Blob never holds it whole. The
     // artifact is content-addressed and immutable, so a deferred read sees
     // the same bytes; if a concurrent `vx cache prune` removed it first the
-    // read throws and this upload is skipped — the never-fail contract.
+    // plugin's read throws and this upload is skipped — the never-fail
+    // contract.
     const hash = args.hash
     const durationMs = args.entry.durationMs
     // Local writes disabled (`--cache=local:,remote:rw`): there is no
@@ -338,16 +470,17 @@ export class LayeredCache implements CacheLayer {
       try {
         packed = await this.local.packArtifactBytes(args)
       } catch (err) {
-        this.reportRemoteError(err)
+        this.reportRemoteError('upload', hash, err)
         return
       }
     }
     this.enqueueUpload(async () => {
       try {
-        const bytes = packed ?? (await Bun.file(this.local.outputsPath(hash)).bytes())
-        await this.remote.put(hash, bytes, { durationMs })
+        const body =
+          packed !== undefined ? new Blob([packed]) : Bun.file(this.local.outputsPath(hash))
+        await this.remote.put(hash, body, { durationMs })
       } catch (err) {
-        this.reportRemoteError(err)
+        this.reportRemoteError('upload', hash, err)
       }
     })
   }
@@ -384,8 +517,8 @@ export class LayeredCache implements CacheLayer {
     }
   }
 
-  async ingest(hash: string, compressed: Uint8Array, meta: IngestMeta): Promise<void> {
-    await this.local.ingest(hash, compressed, meta)
+  async ingest(hash: string, body: Blob | Response, meta: IngestMeta): Promise<void> {
+    await this.local.ingest(hash, body, meta)
   }
 
   loadOutputFilesBatch(hashes: readonly string[]): Map<string, OutputFileRow[]> {
@@ -401,6 +534,9 @@ export class LayeredCache implements CacheLayer {
   recordOutputDirs(hash: string, projectDir: string, prefixes: readonly string[]): Promise<void> {
     return this.local.recordOutputDirs(hash, projectDir, prefixes)
   }
+  recordOutputStamps(hash: string, projectDir: string, workspaceRoot: string): void {
+    this.local.recordOutputStamps(hash, projectDir, workspaceRoot)
+  }
 
   loadOutputDirsBatch(hashes: readonly string[]): Map<string, OutputDirRow[]> {
     return this.local.loadOutputDirsBatch(hashes)
@@ -410,14 +546,6 @@ export class LayeredCache implements CacheLayer {
     return this.local.outputDirsCurrent(projectDir, rows)
   }
 
-  recordRun(run: RunRecord): void {
-    this.local.recordRun(run)
-  }
-
-  recordRuns(runs: readonly RunRecord[]): void {
-    this.local.recordRuns(runs)
-  }
-
   recordRunBundle(bundle: { runs: readonly RunRecord[]; invocation: InvocationRecord }): void {
     this.local.recordRunBundle(bundle)
   }
@@ -426,16 +554,46 @@ export class LayeredCache implements CacheLayer {
     return this.local.stats(opts)
   }
 
-  async prune(options: PruneOptions): Promise<PruneResult> {
-    return await this.local.prune(options)
+  prune(options: PruneOptions): Promise<PruneResult> {
+    return this.local.prune(options)
   }
 
+  /** `run()` drains the uploads and prefetches first, so the repeat counts are final here. */
   close(): void {
+    for (const { cause, repeats } of this.reported.values()) {
+      if (repeats === 0) continue
+      this.emit(
+        new Error(
+          `${repeats} more request${repeats === 1 ? '' : 's'} failed the same way: ${cause}`,
+        ),
+      )
+    }
+    this.reported.clear()
     this.local.close()
   }
 
-  private reportRemoteError(err: unknown): void {
+  /**
+   * `subject` is the artifact's hash, or the batch for a `hasMany`. The
+   * wire's own message says what went wrong but not on what or where: an
+   * upload deadline read `The operation timed out.` with no upload, hash or
+   * server in it (item 749).
+   */
+  private reportRemoteError(op: RemoteOp, subject: string | { batch: string }, err: unknown): void {
     const e = err instanceof Error ? err : new Error(String(err))
+    const hash = typeof subject === 'string' ? subject : undefined
+    const cls = failureClass(e, hash)
+    const seen = this.reported.get(cls)
+    if (seen !== undefined) {
+      seen.repeats++
+      return
+    }
+    this.reported.set(cls, { cause: e.message, repeats: 0 })
+    const what = typeof subject === 'string' ? subject : subject.batch
+    const where = this.endpoint === undefined ? '' : ` ${PREPOSITION[op]} ${this.endpoint}`
+    this.emit(new Error(`${op} ${what}${where} failed: ${e.message}`, { cause: e }))
+  }
+
+  private emit(e: Error): void {
     // The remote cache is fully optional: NO remote failure — a 500, a
     // network drop, a corrupt artifact, even a throwing onRemoteError
     // callback — may ever fail the run. We report and degrade to a
@@ -452,3 +610,7 @@ export class LayeredCache implements CacheLayer {
     }
   }
 }
+
+function durationOf(ms: unknown): number {
+  return typeof ms === 'number' && ms >= 0 && ms <= Number.MAX_SAFE_INTEGER ? Math.round(ms) : 0
+}