@vzn/vx-reapi 0.0.652 → 0.0.653

package/README.md

@@ -74,14 +74,15 @@ A CAS digest is the sha256 of the **content**, so it cannot be derived from a
 vx cache key before the bytes exist — `has(key)` could never answer. The
 ActionCache supplies the missing indirection:
 
-| vx                   | REAPI                                                                                                                                                            |
-| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| cache key            | synthetic action digest — `sha256("vx-reapi-v1\0" + key)`                                                                                                        |
-| artifact (`tar.zst`) | one CAS blob, referenced by the ActionResult's `output_files`                                                                                                    |
-| task duration        | `stdout_raw` on the ActionResult                                                                                                                                 |
-| cache miss           | `GetActionResult` → `NOT_FOUND`                                                                                                                                  |
-| cache hit            | `GetActionResult` asking for stdout and the artifact inline: a server that honours it (bazel-remote, up to ~1 MiB) answers a small hit in one round trip         |
-| cache save           | an artifact up to 256 KiB is read once and goes in one `BatchUpdateBlobs`, unprobed; a larger one is probed with `FindMissingBlobs` and streamed only if missing |
+| vx                   | REAPI                                                                                                                                                                  |
+| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| cache key            | synthetic action digest — `sha256("vx-reapi-v1\0" + key)`                                                                                                              |
+| artifact (`tar.zst`) | one CAS blob, referenced by the ActionResult's `output_files`                                                                                                          |
+| task duration        | `stdout_raw` on the ActionResult                                                                                                                                       |
+| cache miss           | `GetActionResult` → `NOT_FOUND`                                                                                                                                        |
+| cache hit            | `GetActionResult` asking for stdout and the artifact inline: a server that honours it (bazel-remote, up to ~1 MiB) answers a small hit in one round trip               |
+| cache save           | an artifact up to 256 KiB is read once and goes in one `BatchUpdateBlobs`, unprobed; a larger one is probed with `FindMissingBlobs` and streamed only if missing       |
+| stored key           | kept: the save writes no `UpdateActionResult`, unless core says the stored bytes did not read back (`replace`); then the blob is sent unprobed and the entry rewritten |
 
 The `vx-reapi-v1` prefix does two jobs: it keeps vx keys out of the address
 space of real Bazel action digests on a shared server, and it makes a future
@@ -132,13 +133,19 @@ bazel-remote sends when the call's `grpc-timeout` runs out first (a
 
 ## Repeat runs skip the worker
 
-Every successful remote execution writes an execution record under the
-task's vx key (`vx-reapi-exec-v1`), listing its outputs by digest plus
-its stdout (inline up to 64 KiB, a CAS blob past it), written while the
-outputs come down. A later run whose vx cache missed but whose key already has
-a record skips the Merkle build, the upload pass and `Execute`
-entirely: the outputs are already in the CAS, and stdout replays from
-the record. `--force` bypasses it.
+Every successful remote execution writes an execution
+record under the task's vx key (`vx-reapi-exec-v2`), listing its outputs by digest plus
+its stdout and stderr (each inline up to 64 KiB, a CAS blob past it), written
+while the outputs come down. A later run whose vx cache missed but whose key
+already has a record skips the Merkle build, the upload pass and `Execute`
+entirely: the outputs are already in the CAS, and both streams replay from
+the record, as a vx hit replays them. A run core does not save
+(`--no-cache`, a read-only policy, a tainted upstream) records under a key
+only this run's executor knows instead: its dependants in this run graft
+from it, and no later run replays it as the task's hit. A `v1` record kept no stderr and is
+never read: its key executes once more and writes a `v2` one. A record
+with a non-zero exit code (vx writes none, but the action cache is the
+server's) is no record either: the task executes. `--force` bypasses it.
 
 The input tree is read after the key was taken, so each file's bytes are
 held to the git blob id the key hashed for it. A file edited, removed or
@@ -153,11 +160,11 @@ listed input (`@vzn/vx-lockfile` lists it on every cached task).
 This matters most under `--cache-download=none`, where deferral leaves no
 local cache entry behind, so vx's own probe misses on every later run
 and the record is what makes the second run cheap. Records are checked
-against the CAS first (`FindMissingBlobs`, the outputs and a stdout blob) — the action cache and the
+against the CAS first (`FindMissingBlobs`, the outputs and any stream blob) — the action cache and the
 CAS evict independently, so a record that outlived its blobs falls
 through to a real execution rather than "succeeding" with nothing.
 The replay itself is a cache read too: a transport failure reading the
-record, its stdout or any output warns and executes, after removing
+record, a stream or any output warns and executes, after removing
 whatever the half-done replay created (core cleaned the outputs once,
 before the replay, and the real run's result must not inherit them).
 The probe sees a Tree by its own digest, not the blobs inside it, so
@@ -177,7 +184,10 @@ under a key asserting those bytes were present. Which upstream bytes a
 command actually reads is unknowable — that is what `dependsOn`
 declares — so the refusal is the only sound reading, and it matches
 what core does when a deferred producer cannot be materialised.
-Re-run the upstream (`--force`) to repopulate the store.
+Re-run the upstream (`--force`) to repopulate the store. An upstream with
+no record at all and its outputs nowhere on this machine (its outputs
+listed in the local index but gone, or a deferred result no record could
+hold) is refused the same way.
 
 Bringing a task's OWN outputs back gets the same treatment. Core's
 contract is that once an executor returns, the declared outputs are on
@@ -225,8 +235,10 @@ content-addressed store: a corrupt or poisoned remote degrades to a miss
 (the cache invariant), never to wrong bytes under a trusted name. Uploads
 were always server-verified; this is the mirror on the read side, the same
 check Bazel's client performs. A streamed read (the cache artifact) is
-hashed as its bytes pass and errors at its end on a mismatch, so vx's
-ingest fails and the hit is a miss. The size is held as the bytes arrive:
+hashed as its bytes pass and errors at its end on a mismatch with core's
+`CorruptArtifactError`, so the hit is a miss and the rebuild's save
+replaces the entry and its blob: one corrupt entry is one rebuild, not
+one on every run. The size is held as the bytes arrive:
 a body past its digest's size is refused at the byte that passes it, a
 zstd reply is decoded no further than that size, and a batch entry for a
 digest not asked for is dropped.
@@ -239,8 +251,12 @@ directory that resolves out through a link, and a file or link that
 lands, through one, where no declared glob names are refused before anything
 is written through them; a link standing at an output file is replaced, never written
 through; a Tree file's setuid and setgid bits are dropped. A Tree may name
-one subtree many times; one that expands to more paths than the file
-system has free inodes is refused before anything is written.
+one subtree many times; a result whose Trees together expand to more paths
+than the file system has free inodes is refused before anything is written,
+as is a Tree holding a path past `PATH_MAX` (4096 bytes; 1024 on macOS).
+A glob's matches are recorded each as a Tree of its own, a subtree named
+under many names encoded once; a capture past the free inodes or
+`PATH_MAX` is recorded whole, unsplit.
 
 ## Artifacts stream
 
@@ -307,19 +323,34 @@ A task placed `exec.remote: 'only'` fails with that reason instead. Unset,
 the task's own `exec.timeout` bounds the queue as it bounds the run (the
 task fails as timed out); with neither, the wait is unbounded.
 A stream that drops with a transient status, or ends cleanly before its
-operation is done, re-attaches with `WaitExecution` (three times, backing
-off 100, 400 and 1600 ms) rather than running the action again. If
+operation is done, re-attaches with `WaitExecution` rather than running the
+action again. Only attaches that deliver no operation spend the budget:
+the fourth such in a row fails the task. A stream that delivered one (a
+proxy cutting a long, progressing action) starts the count over, so a
+long action survives any number of cuts. The wait before each re-attach
+grows per cut in a row (100, 400, 1600, then 5000 ms) and starts over once
+a stream advances the stage or stays open 10 s. If
 `WaitExecution` answers NOT_FOUND — the server lost the operation, as a
 restart does — nothing is left running to re-attach to, and the action is
-executed again on the same budget.
+executed again on the same budget. Past the budget the task fails and the
+operation is cancelled, so the action does not run on unwatched.
+An Execute answered FAILED_PRECONDITION (inputs the server lacks, evicted
+after `FindMissingBlobs` or never stored) uploads them again — the blobs its
+`PreconditionFailure` names MISSING, else every input, without asking
+`FindMissingBlobs` — and executes once more; a second FAILED_PRECONDITION
+fails the task.
 Once a worker reports EXECUTING, the task's `exec.timeout` (or
-`executeTimeoutMs`) bounds the wait, and the bound holds across a
-re-attach: firing during the backoff between a dropped stream and its
-`WaitExecution`, it ends the task there rather than being lost. Past it
-the operation is cancelled too (a refused cancel is a warning).
+`executeTimeoutMs`, 10 minutes when neither is set) bounds the wait, and
+the bound holds across a re-attach: firing during the backoff between a
+dropped stream and its `WaitExecution`, it ends the task there rather than
+being lost. It is also what ends a server that answers EXECUTING and hangs
+up on every attach. Past it the operation is cancelled too (a refused
+cancel is a warning).
 The run stopping (Ctrl-C, an embedder's abort) or the task's `exec.timeout`
 closes the stream and cancels the operation with `CancelOperation`, and an
-action not yet submitted is not sent. The task then ends as a killed local
+action not yet submitted is not sent. A record replay, an upstream record
+read or the upload under way stops there too, and writes nothing after: what
+a replay already created is taken back, so a retry starts clean. The task then ends as a killed local
 one does (exit 143, `SIGTERM`): past `exec.timeout` it is `failed (timed
 out)` with its key in `vx last`.
 
@@ -349,8 +380,9 @@ and the task executes normally, with a warning naming why. The UPSTREAM record
 reads are the deliberate exception — those decide whether a dependency's bytes
 exist at all, and carrying on past a failure there is how an action runs
 without its inputs and caches the result. A fresh execution's stdout or
-stderr that cannot be read fails the task, as a declared output does
-(above): those are the task's result, not a shortcut.
+stderr that cannot be read, or whose blob the CAS no longer holds, fails
+the task naming the digest, and no record is written: those are the
+task's result, not a shortcut.
 
 ## Tests
 
@@ -428,7 +460,10 @@ A cache-only server (bazel-remote advertises `exec_enabled: false`) makes the
 plugin **decline the executor with a warning** rather than submit work that
 will never be answered. Only cacheable tasks are eligible — a task with no
 `cache: true` has no described inputs, so a worker would run it against an
-empty input root. A cacheable task's root holds its declared input files.
+empty input root. A cacheable task's root holds its declared input files
+and its upstreams' declared outputs. An upstream with no `cache` runs on
+this machine and leaves no record, so its outputs ship from this disk; a
+file it writes outside its declared `outputs` never reaches the worker.
 
 Verified end-to-end against a live NativeLink scheduler + worker: input tree
 uploaded, QUEUED → EXECUTING → COMPLETED streamed, stdout returned inline,
@@ -480,11 +515,15 @@ worker (`tests/vx-run-e2e.test.ts`):
   miss can leave the artifact store and the execution record holding
   results of different executions under one pure-input key, and a worker
   fed the record would see bytes this machine's own tasks do not.
+- An upstream that writes inside `install`'s directory (a `prisma generate`
+  into `node_modules/.prisma`) is merged into it: the worker sees both. A
+  DECLARED input file under a grafted directory is replaced by the graft,
+  with a warning.
 - With **no remote executor declared**, `install` is a local no-op and
   dependents use whatever the machine has ambient — a laptop run behaves
   exactly as it did before the field existed.
 
-The execution record lives under `sha256("vx-reapi-exec-v1\0" + key)` — a
+The execution record lives under `sha256("vx-reapi-exec-v2\0" + key)` — a
 second AC namespace beside the artifact mapping, listing outputs file-by-file
 with workspace-relative paths.
 
@@ -507,6 +546,10 @@ sorted by name — the proto requires that, so equivalent Commands hash alike:
 A `define` wins over an `inputs.env` entry of the same name: it is the more
 explicit statement of intent.
 
+One constant is vx's own: `FORCE_COLOR=1`, which every local child gets
+too (unless the task sees a `FORCE_COLOR` or a non-empty `NO_COLOR`), so a
+remote entry holds the same coloured bytes a local one does.
+
 Nothing else crosses. In particular **`exec.env.passThrough` does not reach a
 remote worker**, and neither does the essential allowlist (`PATH`, `HOME`,
 `TMPDIR`, …) — those are the submitting machine's resolved environment, and

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vzn/vx-reapi",
-  "version": "0.0.652",
+  "version": "0.0.653",
   "license": "MIT",
   "description": "Bazel Remote Execution API plugin for vx — remote cache (ActionCache + CAS) against NativeLink, BuildBuddy, Buildbarn or bazel-remote.",
   "keywords": [
@@ -39,7 +39,7 @@
     "access": "public"
   },
   "peerDependencies": {
-    "@vzn/vx": "^0.0.652"
+    "@vzn/vx": "^0.0.653"
   },
   "repository": {
     "type": "git",

package/src/cache.ts

@@ -11,7 +11,7 @@
 import { createHash } from 'node:crypto'
 import * as grpc from '@grpc/grpc-js'
 import { type ActionResult, type Digest, ReapiClient, type ReapiOptions } from './wire.js'
-import { ui } from '@vzn/vx'
+import { CorruptArtifactError, ui } from '@vzn/vx'
 
 /** The artifact's `output_files` path. Constant: the AC entry holds exactly one. */
 const ARTIFACT_PATH = 'vx-artifact.tar.zst'
@@ -36,7 +36,9 @@ export function actionDigestFor(vxKey: string): Digest {
  * without ever landing on the submitter's disk.
  */
 export function execDigestFor(vxKey: string): Digest {
-  const payload = Buffer.from(`vx-reapi-exec-v1\0${vxKey}`, 'utf8')
+  // v2 records keep stderr. A v1 record cannot tell "printed nothing" from
+  // "not kept", so it is never read: its key executes once more.
+  const payload = Buffer.from(`vx-reapi-exec-v2\0${vxKey}`, 'utf8')
   return { hash: createHash('sha256').update(payload).digest('hex'), size_bytes: payload.length }
 }
 
@@ -173,9 +175,16 @@ export class ReapiRemoteCache {
       digestOf(file.contents).hash === file.digest.hash
         ? file.contents
         : undefined
+    // Bytes that are not their digest are wrong on the server, not cut
+    // short: said as core's corrupt artifact, the rebuild's save replaces them.
     const [durationMs, body] = await Promise.all([
       this.durationOf(result),
-      inline === undefined ? this.client.readBlobStream(file.digest) : inline,
+      inline === undefined
+        ? this.client.readBlobStream(
+            file.digest,
+            (err) => new CorruptArtifactError(hash, err.message, err),
+          )
+        : inline,
     ])
     if (body === null) return null
     return { body: new Response(body), durationMs }
@@ -187,7 +196,7 @@ export class ReapiRemoteCache {
     }
     // The server normalised our inline bytes into CAS (bazel-remote does).
     // An absent digest arrives as `null` on this path (proto-loader's
-    // message default, as `this_readStream` in executor.ts records).
+    // message default, as `readStream` in executor.ts records).
     // The duration is metadata: a read of it that fails (retries spent, a
     // blob failing its digest) leaves it unknown, where it made core drop a
     // valid hit as a miss (F-21).
@@ -213,22 +222,27 @@ export class ReapiRemoteCache {
    * An artifact past `writeBlob`'s whole read takes two passes over `body`,
    * never one in memory: the digest first (the CAS address must be known
    * before the server is asked), then the upload from a second read of the
-   * stream.
+   * stream. `replace`: the stored entry did not read back, so the blob is
+   * sent whether the server says it holds it or not, and the entry
+   * rewritten.
    */
-  async put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void> {
+  async put(hash: string, body: Blob, meta: { durationMs: number; replace?: true }): Promise<void> {
     if (this.refused !== undefined) throw this.refused
     // An entry is immutable, and UpdateActionResult replaces one: the
     // lookup runs beside the upload (CAS writes are idempotent), so a key
     // stored already costs no round trip of its own and keeps its result.
     // A lookup that fails writes, as before.
     const action = actionDigestFor(hash)
-    const stored = this.client.getActionResult(action).then(
-      (r) => r?.output_files?.find((f) => f.path === ARTIFACT_PATH)?.digest,
-      (err: unknown) => {
-        this.warn('looking up a stored entry before an upload', err)
-        return undefined
-      },
-    )
+    const stored =
+      meta.replace === true
+        ? undefined
+        : this.client.getActionResult(action).then(
+            (r) => r?.output_files?.find((f) => f.path === ARTIFACT_PATH)?.digest,
+            (err: unknown) => {
+              this.warn('looking up a stored entry before an upload', err)
+              return undefined
+            },
+          )
     // Read once, and those bytes hashed and sent, wherever `writeBlob` would
     // read the Blob whole anyway: a second writer of the key renames its
     // artifact over the file, and two reads sent its bytes under this one's
@@ -248,7 +262,9 @@ export class ReapiRemoteCache {
     } else {
       // Upload only what the server lacks: for a large artifact the probe is
       // cheap next to the bytes it can skip.
-      const missing = await this.client.findMissingBlobs([digest])
+      // A blob held under this digest may be the one that did not read back.
+      const missing =
+        meta.replace === true ? [digest] : await this.client.findMissingBlobs([digest])
       if (missing.length > 0) await this.client.writeBlob(digest, data ?? body)
     }
     // The two stores are pruned apart: an entry whose blob is gone is a