@vzn/vx-reapi 0.0.646 → 0.0.648

package/README.md

@@ -25,7 +25,9 @@ export default defineWorkspace({
 Off CI a run reads the remote and does not write it unless its policy names
 the remote. Give the write credential only to the default branch's CI and a
 read-only one to laptops and pull request jobs: the server's token is the
-boundary (`docs/security.md` § Cache poisoning).
+boundary (`docs/security.md` § Cache poisoning). A server that refuses a
+save's `UpdateActionResult` (`PERMISSION_DENIED`, `UNIMPLEMENTED`) is taken
+at its word for the run: later saves fail the same way without uploading.
 
 `ReapiRemoteCache` is the layer class behind `reapi()`, for a workspace that
 composes cache layers by hand. The package exports `reapi`, `ReapiPluginOptions`,
@@ -37,19 +39,22 @@ and `onWarn` for a degraded-but-recovered call; `reapi()` refuses those four,
 reading files and warning through vx. With no endpoint configured (or a blank
 one) the plugin **declines** and costs nothing, so it is
 safe to leave declared. An endpoint that is not `host[:port]`, with an
-optional `grpc(s)://` or `http(s)://` scheme, or a gRPC resolver target
+optional `grpc(s)://` or `http(s)://` scheme (a trailing `/` is dropped), or a gRPC resolver target
 (`unix:`, `unix-abstract:`, `dns:`, `ipv4:`, `ipv6:`), is refused at startup
 with a line naming the setting. So is a TLS endpoint at an IP address:
 Bun's TLS refuses an IP as the server name, so name the server by host
-name, or use `grpc://` for plaintext. `VX_REAPI_ENDPOINT` / `VX_REAPI_INSTANCE` configure it
-from the environment, and `VX_REAPI_EXECUTE=1` turns on remote execution the
-way `execute: true` does (off by default: a plugin must not move where a
-build runs merely by being configured for caching). `execute` is a
-boolean: a string (`process.env.X`) is refused.
-`instanceName` is the option form of `VX_REAPI_INSTANCE`; like the endpoint
-it is trimmed, and a blank one is unset. `headers`
+name, or use `grpc://` for plaintext. The plugin reads no environment
+variable: every setting is an option, and a value from CI is named in
+`.vx/workspace.ts` (`endpoint: process.env.REAPI_ENDPOINT`). `execute:
+true` turns on remote execution (off by default: a plugin must not move
+where a build runs merely by being configured for caching). `execute` is
+a boolean: a string (`process.env.X`) is refused. `instanceName`, like
+the endpoint, is trimmed, and a blank one is unset. `headers`
 adds gRPC metadata to every call: a hosted server's API key goes there
-(`headers: { 'x-buildbuddy-api-key': process.env.BB_KEY! }`).
+(`headers: { 'x-buildbuddy-api-key': process.env.BB_KEY! }`); a name
+gRPC metadata cannot carry (a space, a `-bin` suffix) is refused at startup,
+and a header or `platform` value that is no string (an unset variable) as
+`reapi()` is called.
 `toolName` and `toolVersion` (default `vx`, `0.0.0`) fill each call's
 `RequestMetadata.tool_details`, and `correlatedInvocationsId` groups several
 runs as one build in a server's UI.
@@ -58,10 +63,8 @@ TLS is on unless asked off, as in Bazel: a bare `host:port`, a `grpcs://`
 or `https://` endpoint, and a `dns:`/`ipv4:`/`ipv6:` target all use it.
 Plaintext takes a `grpc://` or `http://` endpoint or `tls: false`; a `unix:`
 or `unix-abstract:` socket is plaintext (`tls: true` turns it on). It uses the system roots unless told otherwise. A server behind a private
-CA takes `tlsCertificate` (or `VX_REAPI_TLS_CERTIFICATE`), a PEM file of
-that CA; one that asks for mutual TLS takes `tlsClientCertificate` and
-`tlsClientKey` (`VX_REAPI_TLS_CLIENT_CERTIFICATE` / `VX_REAPI_TLS_CLIENT_KEY`)
-— Bazel's `--tls_certificate`, `--tls_client_certificate` and
+CA takes `tlsCertificate`, a PEM file of that CA; one that asks for
+mutual TLS takes `tlsClientCertificate` and `tlsClientKey` — Bazel's `--tls_certificate`, `--tls_client_certificate` and
 `--tls_client_key`. Any of them turns TLS on unless `tls: false` is set; a file that cannot be read
 is refused at startup, naming the setting.
 
@@ -102,9 +105,10 @@ See Bun [#30342](https://github.com/oven-sh/bun/issues/30342) and
 [#26915](https://github.com/oven-sh/bun/issues/26915), largely fixed by
 [#31584](https://github.com/oven-sh/bun/pull/31584) — which is why the ceiling
 _rose_ from ~64 KB on Bun 1.3.x to ~216 KB on 1.4.0 rather than the hang going
-away. Hence **Bun >= 1.4** is required, and the plugin refuses to start on
-anything older with a named error: the alternative is a wedged upload with
-nothing for a user to act on.
+away. Hence **Bun >= 1.4.4** is required (core's own minimum, the package's
+`engines.bun`), and the plugin refuses to start on anything older with a
+named error: the alternative is a wedged upload with nothing for a user to
+act on.
 
 The default is the one size with no peer-dependence — 65535, the RFC 7540
 default initial window every peer must honour with no `WINDOW_UPDATE` at
@@ -114,7 +118,7 @@ the call's 30 s deadline; 65535 stalled in none, and costs ~51% on a 32 MiB
 upload (390 → 590 ms on loopback). A larger `chunkBytes` is still accepted:
 
 ```ts
-reapi({ endpoint: '…', chunkBytes: 128 * 1024 })
+reapi({ endpoint: 'grpcs://cache.example.com:443', chunkBytes: 128 * 1024 })
 ```
 
 The stall is a RACE, not a boundary. So above the safe size the client
@@ -137,18 +141,19 @@ entirely: the outputs are already in the CAS, and stdout replays from
 the record. `--force` bypasses it.
 
 The input tree is read after the key was taken, so each file's bytes are
-held to the git blob id the key folded for it. A file edited or removed
+held to the git blob id the key hashed for it. A file edited, removed or
+replaced by a directory
 in between (an edit mid-run under `vx watch`) still runs, but the
 execution is not recorded under a key that no longer describes it; the
 record once replayed the edited outputs on every machine after the file
-was restored. The input root holds exactly the files the key folded: core
+was restored. The input root holds exactly the files the key hashed: core
 keys no package file, so a project's `package.json` is in it only as a
 listed input (`@vzn/vx-lockfile` lists it on every cached task).
 
 This matters most under `--cache-download=none`, where deferral leaves no
 local cache entry behind, so vx's own probe misses on every later run
 and the record is what makes the second run cheap. Records are checked
-against the CAS first (`FindMissingBlobs`) — the action cache and the
+against the CAS first (`FindMissingBlobs`, the outputs and a stdout blob) — the action cache and the
 CAS evict independently, so a record that outlived its blobs falls
 through to a real execution rather than "succeeding" with nothing.
 The replay itself is a cache read too: a transport failure reading the
@@ -165,7 +170,7 @@ An UPSTREAM's evicted blobs get the opposite answer, because there is
 nothing to fall through to. When a dependency's outputs live only in
 the CAS — vx grafts them by reference precisely because no local copy
 exists — and those blobs are gone, the action cannot be built with the
-inputs its key claims. vx fails the task and names the upstream rather
+inputs its key names. vx fails the task and names the upstream rather
 than shipping the action without them: a command that tolerates the
 absence exits 0, and that successful-but-wrong result would be cached
 under a key asserting those bytes were present. Which upstream bytes a
@@ -178,7 +183,7 @@ Bringing a task's OWN outputs back gets the same treatment. Core's
 contract is that once an executor returns, the declared outputs are on
 disk, because the ordinary save path then tars whatever it finds — so
 an output blob that cannot be fetched is a hole that would be cached
-under a key claiming a complete build. Under a literal capture the
+under a key that stands for a complete build. Under a literal capture the
 worker returns only what `output_paths` named, so every returned file
 is a declared output and an unfetchable one fails the task. The
 exception is a glob whose FIRST segment is a wildcard (`*.js`): it has
@@ -229,10 +234,13 @@ digest not asked for is dropped.
 A verified blob still lands where the server's ActionResult says, so its
 paths are held to the workspace: an output path or Tree name that climbs
 out (`..`, absolute), a link whose target leaves the workspace (read as
-the OS follows it, through the links the result placed), and a
-directory that resolves out through a link are refused before anything
+the OS follows it, through the links the result placed), a
+directory that resolves out through a link, and a file or link that
+lands, through one, where no declared glob names are refused before anything
 is written through them; a link standing at an output file is replaced, never written
-through; a Tree file's setuid and setgid bits are dropped.
+through; a Tree file's setuid and setgid bits are dropped. A Tree may name
+one subtree many times; one that expands to more paths than the file
+system has free inodes is refused before anything is written.
 
 ## Artifacts stream
 
@@ -268,10 +276,9 @@ buying headroom for that upload also buys every metadata probe the same
 minutes before it can degrade, which is the opposite of what the deadline is
 for.
 
-Each deadline must be a positive number of ms; anything else is refused when
-the plugin starts. `executeTimeoutMs` and `queueTimeoutMs` are held to the
-same rule when the executor starts, so with `execute` off they are not
-checked.
+Each deadline, `executeTimeoutMs` and `queueTimeoutMs` among them, must be
+a positive number of ms up to 2147483647 (a longer timer fires at once);
+anything else is refused at the `reapi()` call.
 
 That is not hypothetical. A NativeLink instance degraded into a state where it
 answered every ActionCache MISS in 3 ms and every HIT never — idle CPU,
@@ -280,11 +287,11 @@ single 180 s deadline, every task burned three minutes on a lookup before
 failing. Now the probe gives up in 15 s and the run re-executes.
 
 What the run prints is one line per kind of failure:
-`vx/reapi: probe <key> at <endpoint> failed: 4 DEADLINE_EXCEEDED: …`.
+`[vx] plugin '@vzn/vx-reapi': probe <key> at <endpoint> failed: 4 DEADLINE_EXCEEDED: …`.
 Core's layered cache names the request, the vx key and the server (a
 gRPC status carries none of them), and a later request that fails with
 the same status is counted, not repeated: the run ends with
-`vx/reapi: N more requests failed the same way: …`.
+`[vx] plugin '@vzn/vx-reapi': N more requests failed the same way: …`.
 
 Execution streams are deliberately NOT bounded by either: queueing behind a
 busy worker pool is legitimate. A wedged server still cannot reach Execute,
@@ -293,9 +300,9 @@ worker is bounded by `queueTimeoutMs` when set: no EXECUTING within it, the
 Execute stream is closed, the operation is cancelled with
 `Operations.CancelOperation` (one attempt on `metaTimeoutMs`; a server
 that refuses it, or lacks the service, may still run the action, and its
-result lands in the action cache) and the task is given back to vx, which
+result lands in the action cache; that refusal is a warning) and the task is given back to vx, which
 runs it here and says so once:
-`[vx] <task>: vx/reapi: no worker started the action within queueTimeoutMs (…ms); its operation was cancelled — running it here`.
+`[vx] <task>: plugin '@vzn/vx-reapi' (executor 'vx/reapi') gave it back: no worker started the action within queueTimeoutMs (…ms); its operation was cancelled — running it here`.
 A task placed `exec.remote: 'only'` fails with that reason instead. Unset,
 the task's own `exec.timeout` bounds the queue as it bounds the run (the
 task fails as timed out); with neither, the wait is unbounded.
@@ -308,7 +315,8 @@ executed again on the same budget.
 Once a worker reports EXECUTING, the task's `exec.timeout` (or
 `executeTimeoutMs`) bounds the wait, and the bound holds across a
 re-attach: firing during the backoff between a dropped stream and its
-`WaitExecution`, it ends the task there rather than being lost.
+`WaitExecution`, it ends the task there rather than being lost. Past it
+the operation is cancelled too (a refused cancel is a warning).
 The run stopping (Ctrl-C, an embedder's abort) or the task's `exec.timeout`
 closes the stream and cancels the operation with `CancelOperation`, and an
 action not yet submitted is not sent. The task then ends as a killed local
@@ -335,12 +343,14 @@ A status the server answers with and no retry heals (PERMISSION_DENIED on
 Execute, a refused upload) fails the task with a line naming the task and
 the status, never as a vx "internal error".
 
-A failed READ is never a failed task. The execution-record lookup is a
+A failed RECORD read is never a failed task. The execution-record lookup is a
 shortcut past the worker, so a transport error there means "no usable record"
 and the task executes normally, with a warning naming why. The UPSTREAM record
 reads are the deliberate exception — those decide whether a dependency's bytes
 exist at all, and carrying on past a failure there is how an action runs
-without its inputs and caches the result.
+without its inputs and caches the result. A fresh execution's stdout or
+stderr that cannot be read fails the task, as a declared output does
+(above): those are the task's result, not a shortcut.
 
 ## Tests
 
@@ -348,7 +358,7 @@ without its inputs and caches the result.
 servers, which `vx run test.live --filter @vzn/vx-reapi` brings with Docker:
 `services.cache` (bazel-remote on `:19092`) and `services.exec` (NativeLink on
 `:51051`) run in the foreground until the run ends. CI runs it in the Linux
-distributed run, where they start on the agent that runs `test.live`.
+DTE run, where they start on the agent that runs `test.live`.
 
 Without an endpoint those tests skip; `test.live` sets `VX_REQUIRE_REAPI=1` and
 `VX_REQUIRE_REAPI_EXEC=1`, which turn an absent endpoint into a failure so the
@@ -386,13 +396,15 @@ Protocol features in use, not just reachable:
   hung one.
 - **`Action.platform`** (v2.2) alongside `Command.platform` for older
   servers.
-- **`NodeProperties`** — `unix_mode` and `mtime` on tree nodes.
+- **`NodeProperties`** — `unix_mode` and `mtime` on tree nodes; a mode (and
+  `is_executable`) lands as its permission bits minus the process umask.
 - **Output directories** via the `Tree` blob an `OutputDirectory.tree_digest`
   addresses, plus **output symlinks** (a v2.0 server's
   `output_file_symlinks` / `output_directory_symlinks` when it sends only
   those). A tree's small files are fetched
   together across its directories (`BatchReadBlobs`, 64 MiB at a time),
-  not one call per directory.
+  not one call per directory; the result's own output files, 64 MiB at a
+  time too.
 - **Upload minimality** — `FindMissingBlobs` first (split so no request
   passes a 4 MiB message), then batched blobs while they fit the server's
   budget and ByteStream beyond it.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vzn/vx-reapi",
-  "version": "0.0.646",
+  "version": "0.0.648",
   "license": "MIT",
   "description": "Bazel Remote Execution API plugin for vx — remote cache (ActionCache + CAS) against NativeLink, BuildBuddy, Buildbarn or bazel-remote.",
   "keywords": [
@@ -29,7 +29,7 @@
     "LICENSE"
   ],
   "engines": {
-    "bun": ">=1.4"
+    "bun": ">=1.4.4"
   },
   "dependencies": {
     "@grpc/grpc-js": "^1.12.0",
@@ -39,7 +39,7 @@
     "access": "public"
   },
   "peerDependencies": {
-    "@vzn/vx": "^0.0.646"
+    "@vzn/vx": "^0.0.648"
   },
   "repository": {
     "type": "git",

package/src/cache.ts

@@ -9,7 +9,9 @@
 // as a general key/value cache.
 
 import { createHash } from 'node:crypto'
+import * as grpc from '@grpc/grpc-js'
 import { type ActionResult, type Digest, ReapiClient, type ReapiOptions } from './wire.js'
+import { ui } from '@vzn/vx'
 
 /** The artifact's `output_files` path. Constant: the AC entry holds exactly one. */
 const ARTIFACT_PATH = 'vx-artifact.tar.zst'
@@ -61,16 +63,19 @@ function encodeDuration(durationMs: number): Uint8Array {
   return new TextEncoder().encode(JSON.stringify({ durationMs }))
 }
 
+/** The duration a record carries; throws on bytes that are not one. */
 function decodeDuration(raw: Uint8Array | undefined): number | undefined {
   if (raw === undefined || raw.length === 0) return undefined
-  try {
-    const parsed = JSON.parse(new TextDecoder().decode(raw)) as { durationMs?: number }
-    return typeof parsed.durationMs === 'number' ? parsed.durationMs : undefined
-  } catch {
-    return undefined
-  }
+  const parsed = JSON.parse(new TextDecoder().decode(raw)) as { durationMs?: number }
+  return typeof parsed.durationMs === 'number' ? parsed.durationMs : undefined
 }
 
+/**
+ * A duration record vx writes is some 20 bytes; a blob past this is not
+ * one, and is never read: a 256 MiB one was held whole for its number.
+ */
+const DURATION_BYTES = 1024
+
 /** Artifacts up to this size are batch-uploaded without a FindMissingBlobs probe. */
 const SMALL_PUT_BYTES = 256 * 1024
 
@@ -78,10 +83,39 @@ export class ReapiRemoteCache {
   private readonly client: ReapiClient
   /** Named in core's degrade line: a gRPC status carries no server. */
   readonly endpoint: string
+  private readonly onWarn: (message: string) => void
+  /** What was said already: a fault every hit meets is said once a run. */
+  private readonly said = new Set<string>()
+  /**
+   * The action cache's refusal of a write, which stands for the run: a
+   * read-only key or an AC with updates off refuses every save the same
+   * way, and each one uploaded its whole artifact before it was refused.
+   */
+  private refused: Error | undefined
 
   constructor(opts: ReapiOptions) {
     this.client = new ReapiClient(opts)
     this.endpoint = opts.endpoint
+    this.onWarn = opts.onWarn ?? ((m) => ui.say('warn', m))
+  }
+
+  /** A fault the cache works around, said once per kind (2026-10-23: none is silent). */
+  private warn(kind: string, err: unknown): void {
+    if (this.said.has(kind)) return
+    this.said.add(kind)
+    this.onWarn(
+      `[vx] vx-reapi: ${kind} at ${this.endpoint} failed: ${err instanceof Error ? err.message : String(err)}`,
+    )
+  }
+
+  /** The duration a hit carries; one that cannot be read is unknown, and said. */
+  private duration(raw: Uint8Array | undefined): number | undefined {
+    try {
+      return decodeDuration(raw)
+    } catch (err) {
+      this.warn('reading a task duration', err)
+      return undefined
+    }
   }
 
   /**
@@ -92,7 +126,7 @@ export class ReapiRemoteCache {
    * ActionResult's referenced blobs and hides a dangling entry, NativeLink
    * serves it. Without the second call, `has` on a NativeLink-style server
    * promises a hit that `get` then cannot honour — and the only consumer of
-   * `has` is the `--dry` / `--graph` plan, whose entire job is predicting
+   * `has` is the `--dry` plan, whose entire job is predicting
    * hit vs miss. Costs one extra round trip, and only for a PREDICTED HIT:
    * a miss still answers in one call.
    */
@@ -103,7 +137,7 @@ export class ReapiRemoteCache {
     if (file === undefined) return null
     if ((await this.client.findMissingBlobs([file.digest])).length > 0) return null
     // The duration only when it rode inline: a CAS read for it is a round trip.
-    const durationMs = decodeDuration(result.stdout_raw)
+    const durationMs = this.duration(result.stdout_raw)
     return {
       size: Number(file.digest.size_bytes),
       ...(durationMs === undefined ? {} : { durationMs }),
@@ -149,7 +183,7 @@ export class ReapiRemoteCache {
 
   private async durationOf(result: ActionResult): Promise<number | undefined> {
     if (result.stdout_raw !== undefined && result.stdout_raw.length > 0) {
-      return decodeDuration(result.stdout_raw)
+      return this.duration(result.stdout_raw)
     }
     // The server normalised our inline bytes into CAS (bazel-remote does).
     // An absent digest arrives as `null` on this path (proto-loader's
@@ -157,9 +191,20 @@ export class ReapiRemoteCache {
     // The duration is metadata: a read of it that fails (retries spent, a
     // blob failing its digest) leaves it unknown, where it made core drop a
     // valid hit as a miss (F-21).
-    if ((result.stdout_digest?.size_bytes ?? 0) > 0) {
-      const raw = await this.client.readBlob(result.stdout_digest!).catch(() => null)
-      return decodeDuration(raw ?? undefined)
+    const size = Number(result.stdout_digest?.size_bytes ?? 0)
+    if (size > DURATION_BYTES) {
+      this.warn(
+        'reading a task duration',
+        new Error(`its record is ${size} bytes, over ${DURATION_BYTES}: not read`),
+      )
+      return undefined
+    }
+    if (size > 0) {
+      const raw = await this.client.readBlob(result.stdout_digest!).catch((err: unknown) => {
+        this.warn('reading a task duration', err)
+        return null
+      })
+      return this.duration(raw ?? undefined)
     }
     return undefined
   }
@@ -171,14 +216,18 @@ export class ReapiRemoteCache {
    * stream.
    */
   async put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void> {
+    if (this.refused !== undefined) throw this.refused
     // An entry is immutable, and UpdateActionResult replaces one: the
     // lookup runs beside the upload (CAS writes are idempotent), so a key
     // stored already costs no round trip of its own and keeps its result.
     // A lookup that fails writes, as before.
     const action = actionDigestFor(hash)
     const stored = this.client.getActionResult(action).then(
-      (r) => r?.output_files?.some((f) => f.path === ARTIFACT_PATH) === true,
-      () => false,
+      (r) => r?.output_files?.find((f) => f.path === ARTIFACT_PATH)?.digest,
+      (err: unknown) => {
+        this.warn('looking up a stored entry before an upload', err)
+        return undefined
+      },
     )
     // Read once, and those bytes hashed and sent, wherever `writeBlob` would
     // read the Blob whole anyway: a second writer of the key renames its
@@ -192,21 +241,38 @@ export class ReapiRemoteCache {
       // it (102 → 71 ms per save at 15 ms one-way, F-26). Content-addressed,
       // so a re-send is harmless.
       // A server whose batch limit is smaller refuses it; stream instead.
-      await this.client
-        .batchUpdateBlobs([{ digest, data }])
-        .catch(() => this.client.writeBlob(digest, data))
+      await this.client.batchUpdateBlobs([{ digest, data }]).catch((err: unknown) => {
+        this.warn('a batch upload (streamed instead)', err)
+        return this.client.writeBlob(digest, data)
+      })
     } else {
       // Upload only what the server lacks: for a large artifact the probe is
       // cheap next to the bytes it can skip.
       const missing = await this.client.findMissingBlobs([digest])
       if (missing.length > 0) await this.client.writeBlob(digest, data ?? body)
     }
-    if (await stored) return
-    await this.client.updateActionResult(action, {
-      exit_code: 0,
-      output_files: [{ path: ARTIFACT_PATH, digest, is_executable: false }],
-      stdout_raw: encodeDuration(meta.durationMs),
-    })
+    // The two stores are pruned apart: an entry whose blob is gone is a
+    // miss every run, and kept, its key never hit again.
+    const kept = await stored
+    if (
+      kept !== undefined &&
+      (kept.hash === digest.hash || (await this.client.findMissingBlobs([kept])).length === 0)
+    ) {
+      return
+    }
+    await this.client
+      .updateActionResult(action, {
+        exit_code: 0,
+        output_files: [{ path: ARTIFACT_PATH, digest, is_executable: false }],
+        stdout_raw: encodeDuration(meta.durationMs),
+      })
+      .catch((err: unknown) => {
+        const code = (err as { code?: unknown } | null)?.code
+        if (code === grpc.status.PERMISSION_DENIED || code === grpc.status.UNIMPLEMENTED) {
+          this.refused = err as Error
+        }
+        throw err
+      })
   }
 
   close(): void {