@vzn/vx-reapi 0.0.591 → 0.0.595

package/README.md

@@ -223,11 +223,12 @@ through; a Tree file's setuid and setgid bits are dropped.
 ## Artifacts stream
 
 An artifact up to 256 KiB is read whole and sent in one batch, with no
-probe first. Past that the cache layer never holds an artifact whole: `put` digests the file-backed
-`Blob` vx hands it in one pass over its stream, asks `FindMissingBlobs`, and
-uploads from a second pass: past the batch limit (about 4 MiB) the file is
-read `chunkBytes` at a time as the ByteStream write drains, identity-encoded
-(the artifact is zstd already). `get` returns the ByteStream read as a
+probe first. Up to the batch limit (about 4 MiB) it is read whole once,
+probed with `FindMissingBlobs`, and those bytes sent. Past that the cache
+layer never holds an artifact whole: `put` digests the file-backed `Blob`
+vx hands it in one pass over its stream, asks `FindMissingBlobs`, and
+uploads from a second pass, read `chunkBytes` at a time as the ByteStream
+write drains, identity-encoded (the artifact is zstd already). `get` returns the ByteStream read as a
 `Response`, each message taken from the call as vx writes the previous one
 to disk.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vzn/vx-reapi",
-  "version": "0.0.591",
+  "version": "0.0.595",
   "license": "MIT",
   "description": "Bazel Remote Execution API plugin for vx — remote cache (ActionCache + CAS) against NativeLink, BuildBuddy, Buildbarn or bazel-remote.",
   "keywords": [
@@ -33,14 +33,13 @@
   },
   "dependencies": {
     "@grpc/grpc-js": "^1.12.0",
-    "@grpc/proto-loader": "^0.7.13",
-    "protobufjs": "^7.4.0"
+    "@grpc/proto-loader": "^0.7.13"
   },
   "publishConfig": {
     "access": "public"
   },
   "peerDependencies": {
-    "@vzn/vx": "^0.0.591"
+    "@vzn/vx": "^0.0.595"
   },
   "repository": {
     "type": "git",

package/src/cache.ts

@@ -159,32 +159,32 @@ export class ReapiRemoteCache {
   }
 
   /**
-   * A large artifact takes two passes over `body`, never one in memory: the
-   * digest first (the CAS address must be known before the server is asked),
-   * then the upload from a second read of the stream.
+   * An artifact past `writeBlob`'s whole read takes two passes over `body`,
+   * never one in memory: the digest first (the CAS address must be known
+   * before the server is asked), then the upload from a second read of the
+   * stream.
    */
   async put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void> {
-    let digest: Digest
-    if (body.size <= SMALL_PUT_BYTES) {
+    // Read once, and those bytes hashed and sent, wherever `writeBlob` would
+    // read the Blob whole anyway: a second writer of the key renames its
+    // artifact over the file, and two reads sent its bytes under this one's
+    // digest (F-50).
+    const data = body.size <= this.client.wholeWriteBytes ? await body.bytes() : undefined
+    const digest = data === undefined ? await streamedDigestOf(body) : digestOf(data)
+    if (data !== undefined && data.length <= SMALL_PUT_BYTES) {
       // A small artifact is sent in one batch without asking first: the
       // probe was a round trip of its own to save an upload no larger than
       // it (102 → 71 ms per save at 15 ms one-way, F-26). Content-addressed,
       // so a re-send is harmless.
-      // Read once, and those bytes hashed and sent: a second writer of the
-      // key renames its artifact over the file, and two reads sent its bytes
-      // under this one's digest (F-50).
       // A server whose batch limit is smaller refuses it; stream instead.
-      const data = await body.bytes()
-      digest = digestOf(data)
       await this.client
         .batchUpdateBlobs([{ digest, data }])
         .catch(() => this.client.writeBlob(digest, data))
     } else {
-      digest = await streamedDigestOf(body)
       // Upload only what the server lacks: for a large artifact the probe is
       // cheap next to the bytes it can skip.
       const missing = await this.client.findMissingBlobs([digest])
-      if (missing.length > 0) await this.client.writeBlob(digest, body)
+      if (missing.length > 0) await this.client.writeBlob(digest, data ?? body)
     }
     await this.client.updateActionResult(actionDigestFor(hash), {
       exit_code: 0,

package/src/executor.ts

@@ -12,7 +12,14 @@
 import { mkdir, writeFile, chmod, readlink, realpath, rm, symlink, unlink } from 'node:fs/promises'
 import { constants, existsSync } from 'node:fs'
 import path from 'node:path'
-import { executorFallback, isLiteralPattern, isUserError, normalizeGlob, UserError } from '@vzn/vx'
+import {
+  executorFallback,
+  isLiteralPattern,
+  isUserError,
+  normalizeGlob,
+  UserError,
+  withForwardArgs,
+} from '@vzn/vx'
 import type { ExecuteRequest, ExecuteResult, TaskExecutor, TaskPlacement } from '@vzn/vx'
 import {
   buildInputTree,
@@ -30,6 +37,9 @@ import {
 import { execDigestFor } from './cache.js'
 import type { ActionResult, Digest, Directory, Operation, ReapiClient } from './wire.js'
 
+// A default decoder strips a leading U+FEFF, which is part of a path.
+const PATH_UTF8 = new TextDecoder('utf-8', { ignoreBOM: true })
+
 /**
  * Split a coarse output directory into the paths its GLOB actually names.
  *
@@ -346,7 +356,7 @@ function decodeOutputFile(buf: Uint8Array): {
       i = l
       const slice = buf.subarray(i, i + len)
       i += len
-      if (field === 1) out.path = new TextDecoder().decode(slice)
+      if (field === 1) out.path = PATH_UTF8.decode(slice)
       else if (field === 2) out.digest = decodeDigest(slice)
       else if (field === 5 && len > 0) out.contents = slice
     } else if (wire === 0) {
@@ -370,8 +380,8 @@ function decodeOutputSymlink(buf: Uint8Array): { path: string; target: string }
     i = l
     const slice = buf.subarray(i, i + len)
     i += len
-    if (key >>> 3 === 1) out.path = new TextDecoder().decode(slice)
-    else if (key >>> 3 === 2) out.target = new TextDecoder().decode(slice)
+    if (key >>> 3 === 1) out.path = PATH_UTF8.decode(slice)
+    else if (key >>> 3 === 2) out.target = PATH_UTF8.decode(slice)
   }
   return out
 }
@@ -445,7 +455,7 @@ function decodeOutputDirectory(buf: Uint8Array): { path: string; tree_digest: Di
     i = l
     const slice = buf.subarray(i, i + len)
     i += len
-    if (field === 1) out.path = new TextDecoder().decode(slice)
+    if (field === 1) out.path = PATH_UTF8.decode(slice)
     else if (field === 3) out.tree_digest = decodeDigest(slice)
   }
   return out
@@ -1249,12 +1259,9 @@ async function this_readStream(
   return ''
 }
 
-/** Forwarded args are appended shell-quoted, exactly as the local executor does. */
+/** Forwarded args are placed by core's own `withForwardArgs`, as the local executor's are. */
 function fullCommand(req: ExecuteRequest, cdInto: string, projectRel: string): string {
-  const quoted =
-    req.forwardArgs.length === 0
-      ? req.command
-      : `${req.command} ${req.forwardArgs.map((a) => `'${a.replaceAll("'", `'\\''`)}'`).join(' ')}`
+  const command = withForwardArgs(req.command, req.forwardArgs)
   // A remote action gets NO PATH from this machine — sending one would put a
   // host path in the action digest and split every laptop from every runner.
   // But a task's command is normally a package binary (`oxlint`, `tsc`), and
@@ -1282,7 +1289,7 @@ function fullCommand(req: ExecuteRequest, cdInto: string, projectRel: string): s
   return (
     `VX_ROOT=${root}; ${cd}` +
     `export PATH="$VX_ROOT/node_modules/.bin:$PWD/node_modules/.bin:$PATH"; ` +
-    quoted
+    command
   )
 }
 
@@ -1342,7 +1349,7 @@ export function commandEnvironment(
   return [...merged].map(([name, value]) => ({ name, value }))
 }
 
-export interface OutputPathSets {
+interface OutputPathSets {
   /** v2.1+ `output_paths` — deduped, sorted. */
   outputPaths: string[]
   /** v2.0 legacy split: wildcard-free globs are files, prefix-derived are directories. */

package/src/merkle.ts

@@ -14,12 +14,15 @@ import { lstat, readlink } from 'node:fs/promises'
 import path from 'node:path'
 import type { Digest, Directory, DirectoryNode, FileNode, SymlinkNode } from './wire.js'
 
+// A default decoder strips a leading U+FEFF, which is part of a name.
+const PATH_UTF8 = new TextDecoder('utf-8', { ignoreBOM: true })
+
 export interface Blob {
   digest: Digest
   data: Uint8Array
 }
 
-export interface InputTree {
+interface InputTree {
   /** Digest of the root `Directory` — goes in `Action.input_root_digest`. */
   root: Digest
   /** Every blob the server needs: file contents plus the Directory nodes. */
@@ -740,7 +743,7 @@ function decodeFileNode(buf: Uint8Array): FileNode {
       i = l
       const slice = buf.subarray(i, i + len)
       i += len
-      if (field === 1) f.name = new TextDecoder().decode(slice)
+      if (field === 1) f.name = PATH_UTF8.decode(slice)
       else if (field === 2) f.digest = decodeDigestBytes(slice)
       else if (field === 6) f.node_properties = decodeNodeProperties(slice)
     } else if (wire === 0) {
@@ -802,7 +805,7 @@ function decodeDirectoryNode(buf: Uint8Array): DirectoryNode {
     i = l
     const slice = buf.subarray(i, i + len)
     i += len
-    if (key >>> 3 === 1) d.name = new TextDecoder().decode(slice)
+    if (key >>> 3 === 1) d.name = PATH_UTF8.decode(slice)
     else if (key >>> 3 === 2) d.digest = decodeDigestBytes(slice)
   }
   return d
@@ -819,8 +822,8 @@ function decodeSymlinkNode(buf: Uint8Array): SymlinkNode {
     i = l
     const slice = buf.subarray(i, i + len)
     i += len
-    if (key >>> 3 === 1) sl.name = new TextDecoder().decode(slice)
-    else if (key >>> 3 === 2) sl.target = new TextDecoder().decode(slice)
+    if (key >>> 3 === 1) sl.name = PATH_UTF8.decode(slice)
+    else if (key >>> 3 === 2) sl.target = PATH_UTF8.decode(slice)
   }
   return sl
 }

package/src/wire.ts

@@ -82,7 +82,7 @@ export interface Digest {
   size_bytes: number
 }
 
-export interface ServerCapabilities {
+interface ServerCapabilities {
   digestFunctions: string[]
   maxBatchBytes: number
   acUpdateEnabled: boolean
@@ -1002,6 +1002,11 @@ export class ReapiClient {
     )
   }
 
+  /** The largest `Blob` `writeBlob` reads whole rather than streams. */
+  get wholeWriteBytes(): number {
+    return this.negotiatedBatchBytes || SAFE_BATCH_BYTES
+  }
+
   /**
    * Upload a blob via ByteStream, chunked at `chunkBytes`, RESUMING an
    * interrupted identity upload from the server's committed offset
@@ -1017,8 +1022,7 @@ export class ReapiClient {
    * zstd artifact already. A Blob under the limit is small enough to read.
    */
   async writeBlob(digest: Digest, source: Uint8Array | Blob): Promise<void> {
-    const streamed =
-      source instanceof Blob && source.size > (this.negotiatedBatchBytes || SAFE_BATCH_BYTES)
+    const streamed = source instanceof Blob && source.size > this.wholeWriteBytes
     const body = source instanceof Blob && !streamed ? await source.bytes() : source
     const compressed = this.compression && body instanceof Uint8Array
     // REAPI carries compression in the RESOURCE NAME:
@@ -1566,7 +1570,7 @@ export interface Operation {
   metadata?: { type_url?: string; value?: Uint8Array }
 }
 
-export interface ExecuteOptions {
+interface ExecuteOptions {
   /** The operation's name, once the server has given one. */
   onOperation?: (name: string) => void
   /** Default TRUE: vx owns the cache decision, so the server must not re-check its own AC. */