@vzn/vx-reapi 0.0.594 → 0.0.596

package/README.md

@@ -223,11 +223,12 @@ through; a Tree file's setuid and setgid bits are dropped.
 ## Artifacts stream
 
 An artifact up to 256 KiB is read whole and sent in one batch, with no
-probe first. Past that the cache layer never holds an artifact whole: `put` digests the file-backed
-`Blob` vx hands it in one pass over its stream, asks `FindMissingBlobs`, and
-uploads from a second pass: past the batch limit (about 4 MiB) the file is
-read `chunkBytes` at a time as the ByteStream write drains, identity-encoded
-(the artifact is zstd already). `get` returns the ByteStream read as a
+probe first. Up to the batch limit (about 4 MiB) it is read whole once,
+probed with `FindMissingBlobs`, and those bytes sent. Past that the cache
+layer never holds an artifact whole: `put` digests the file-backed `Blob`
+vx hands it in one pass over its stream, asks `FindMissingBlobs`, and
+uploads from a second pass, read `chunkBytes` at a time as the ByteStream
+write drains, identity-encoded (the artifact is zstd already). `get` returns the ByteStream read as a
 `Response`, each message taken from the call as vx writes the previous one
 to disk.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vzn/vx-reapi",
-  "version": "0.0.594",
+  "version": "0.0.596",
   "license": "MIT",
   "description": "Bazel Remote Execution API plugin for vx — remote cache (ActionCache + CAS) against NativeLink, BuildBuddy, Buildbarn or bazel-remote.",
   "keywords": [
@@ -39,7 +39,7 @@
     "access": "public"
   },
   "peerDependencies": {
-    "@vzn/vx": "^0.0.594"
+    "@vzn/vx": "^0.0.596"
   },
   "repository": {
     "type": "git",

package/src/cache.ts

@@ -159,32 +159,32 @@ export class ReapiRemoteCache {
   }
 
   /**
-   * A large artifact takes two passes over `body`, never one in memory: the
-   * digest first (the CAS address must be known before the server is asked),
-   * then the upload from a second read of the stream.
+   * An artifact past `writeBlob`'s whole read takes two passes over `body`,
+   * never one in memory: the digest first (the CAS address must be known
+   * before the server is asked), then the upload from a second read of the
+   * stream.
    */
   async put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void> {
-    let digest: Digest
-    if (body.size <= SMALL_PUT_BYTES) {
+    // Read once, and those bytes hashed and sent, wherever `writeBlob` would
+    // read the Blob whole anyway: a second writer of the key renames its
+    // artifact over the file, and two reads sent its bytes under this one's
+    // digest (F-50).
+    const data = body.size <= this.client.wholeWriteBytes ? await body.bytes() : undefined
+    const digest = data === undefined ? await streamedDigestOf(body) : digestOf(data)
+    if (data !== undefined && data.length <= SMALL_PUT_BYTES) {
       // A small artifact is sent in one batch without asking first: the
       // probe was a round trip of its own to save an upload no larger than
       // it (102 → 71 ms per save at 15 ms one-way, F-26). Content-addressed,
       // so a re-send is harmless.
-      // Read once, and those bytes hashed and sent: a second writer of the
-      // key renames its artifact over the file, and two reads sent its bytes
-      // under this one's digest (F-50).
       // A server whose batch limit is smaller refuses it; stream instead.
-      const data = await body.bytes()
-      digest = digestOf(data)
       await this.client
         .batchUpdateBlobs([{ digest, data }])
         .catch(() => this.client.writeBlob(digest, data))
     } else {
-      digest = await streamedDigestOf(body)
       // Upload only what the server lacks: for a large artifact the probe is
       // cheap next to the bytes it can skip.
       const missing = await this.client.findMissingBlobs([digest])
-      if (missing.length > 0) await this.client.writeBlob(digest, body)
+      if (missing.length > 0) await this.client.writeBlob(digest, data ?? body)
     }
     await this.client.updateActionResult(actionDigestFor(hash), {
       exit_code: 0,

package/src/executor.ts

@@ -12,16 +12,25 @@
 import { mkdir, writeFile, chmod, readlink, realpath, rm, symlink, unlink } from 'node:fs/promises'
 import { constants, existsSync } from 'node:fs'
 import path from 'node:path'
-import { executorFallback, isLiteralPattern, isUserError, normalizeGlob, UserError } from '@vzn/vx'
+import {
+  executorFallback,
+  isLiteralPattern,
+  isUserError,
+  normalizeGlob,
+  UserError,
+  withForwardArgs,
+} from '@vzn/vx'
 import type { ExecuteRequest, ExecuteResult, TaskExecutor, TaskPlacement } from '@vzn/vx'
 import {
   buildInputTree,
+  decodeDigestBytes,
   decodeTreeWithBytes,
   digestWith,
   encodeAction,
   encodeCommand,
   encodeTree,
   mapBounded,
+  readVarintAt,
   sha256,
   type Blob,
   type FileGraft,
@@ -196,12 +205,12 @@ export function decodeExecuteResponseBytes(buf: Uint8Array): DecodedExecuteRespo
   const out: DecodedExecuteResponse = { serverLogs: [] }
   let i = 0
   while (i < buf.length) {
-    const [key, k] = readVarint(buf, i)
+    const [key, k] = readVarintAt(buf, i)
     i = k
     const field = key >>> 3
     const wire = key & 7
     if (wire === 2) {
-      const [len, l] = readVarint(buf, i)
+      const [len, l] = readVarintAt(buf, i)
       i = l
       const slice = buf.subarray(i, i + len)
       i += len
@@ -212,7 +221,7 @@ export function decodeExecuteResponseBytes(buf: Uint8Array): DecodedExecuteRespo
         if (entry !== undefined) out.serverLogs.push(entry)
       } else if (field === 5) out.message = new TextDecoder().decode(slice)
     } else if (wire === 0) {
-      const [v, n] = readVarint(buf, i)
+      const [v, n] = readVarintAt(buf, i)
       i = n
       if (field === 2) out.cachedResult = v === 1
     } else if (wire === 5) i += 4
@@ -227,16 +236,16 @@ function decodeRpcStatus(buf: Uint8Array): { code: number; message: string } {
   const st = { code: 0, message: '' }
   let i = 0
   while (i < buf.length) {
-    const [key, k] = readVarint(buf, i)
+    const [key, k] = readVarintAt(buf, i)
     i = k
     const field = key >>> 3
     const wire = key & 7
     if (wire === 0) {
-      const [v, n] = readVarint(buf, i)
+      const [v, n] = readVarintAt(buf, i)
       i = n
       if (field === 1) st.code = v
     } else if (wire === 2) {
-      const [len, l] = readVarint(buf, i)
+      const [len, l] = readVarintAt(buf, i)
       i = l
       if (field === 2) st.message = new TextDecoder().decode(buf.subarray(i, i + len))
       i += len
@@ -254,10 +263,10 @@ function decodeLogEntry(
   let humanReadable = false
   let i = 0
   while (i < buf.length) {
-    const [key, k] = readVarint(buf, i)
+    const [key, k] = readVarintAt(buf, i)
     i = k
     if ((key & 7) !== 2) break
-    const [len, l] = readVarint(buf, i)
+    const [len, l] = readVarintAt(buf, i)
     i = l
     const slice = buf.subarray(i, i + len)
     i += len
@@ -265,15 +274,15 @@ function decodeLogEntry(
     else if (key >>> 3 === 2) {
       let j = 0
       while (j < slice.length) {
-        const [k2, j2] = readVarint(slice, j)
+        const [k2, j2] = readVarintAt(slice, j)
         j = j2
         if ((k2 & 7) === 2) {
-          const [len2, j3] = readVarint(slice, j)
+          const [len2, j3] = readVarintAt(slice, j)
           j = j3
-          if (k2 >>> 3 === 1) digest = decodeDigest(slice.subarray(j, j + len2))
+          if (k2 >>> 3 === 1) digest = decodeDigestBytes(slice.subarray(j, j + len2))
           j += len2
         } else if ((k2 & 7) === 0) {
-          const [v, j3] = readVarint(slice, j)
+          const [v, j3] = readVarintAt(slice, j)
           j = j3
           if (k2 >>> 3 === 2) humanReadable = v === 1
         } else break
@@ -288,16 +297,16 @@ function decodeActionResult(buf: Uint8Array): ActionResult {
   const legacyLinks: { path: string; target: string }[] = []
   let i = 0
   while (i < buf.length) {
-    const [key, k] = readVarint(buf, i)
+    const [key, k] = readVarintAt(buf, i)
     i = k
     const field = key >>> 3
     const wire = key & 7
     if (wire === 0) {
-      const [v, n] = readVarint(buf, i)
+      const [v, n] = readVarintAt(buf, i)
       i = n
       if (field === 4) res.exit_code = v | 0
     } else if (wire === 2) {
-      const [len, l] = readVarint(buf, i)
+      const [len, l] = readVarintAt(buf, i)
       i = l
       const slice = buf.subarray(i, i + len)
       i += len
@@ -308,9 +317,9 @@ function decodeActionResult(buf: Uint8Array): ActionResult {
       if (field === 2) (res.output_files ??= []).push(decodeOutputFile(slice))
       else if (field === 3) (res.output_directories ??= []).push(decodeOutputDirectory(slice))
       else if (field === 5) res.stdout_raw = slice
-      else if (field === 6) res.stdout_digest = decodeDigest(slice)
+      else if (field === 6) res.stdout_digest = decodeDigestBytes(slice)
       else if (field === 7) res.stderr_raw = slice
-      else if (field === 8) res.stderr_digest = decodeDigest(slice)
+      else if (field === 8) res.stderr_digest = decodeDigestBytes(slice)
       else if (field === 9) res.execution_metadata = decodeExecutedActionMetadata(slice)
       else if (field === 12) (res.output_symlinks ??= []).push(decodeOutputSymlink(slice))
       // output_file_symlinks (10) and output_directory_symlinks (11): all a
@@ -340,20 +349,20 @@ function decodeOutputFile(buf: Uint8Array): {
   }
   let i = 0
   while (i < buf.length) {
-    const [key, k] = readVarint(buf, i)
+    const [key, k] = readVarintAt(buf, i)
     i = k
     const field = key >>> 3
     const wire = key & 7
     if (wire === 2) {
-      const [len, l] = readVarint(buf, i)
+      const [len, l] = readVarintAt(buf, i)
       i = l
       const slice = buf.subarray(i, i + len)
       i += len
       if (field === 1) out.path = PATH_UTF8.decode(slice)
-      else if (field === 2) out.digest = decodeDigest(slice)
+      else if (field === 2) out.digest = decodeDigestBytes(slice)
       else if (field === 5 && len > 0) out.contents = slice
     } else if (wire === 0) {
-      const [v, n] = readVarint(buf, i)
+      const [v, n] = readVarintAt(buf, i)
       i = n
       if (field === 4) out.is_executable = v === 1
     } else break
@@ -366,10 +375,10 @@ function decodeOutputSymlink(buf: Uint8Array): { path: string; target: string }
   const out = { path: '', target: '' }
   let i = 0
   while (i < buf.length) {
-    const [key, k] = readVarint(buf, i)
+    const [key, k] = readVarintAt(buf, i)
     i = k
     if ((key & 7) !== 2) break
-    const [len, l] = readVarint(buf, i)
+    const [len, l] = readVarintAt(buf, i)
     i = l
     const slice = buf.subarray(i, i + len)
     i += len
@@ -391,12 +400,12 @@ function decodeExecutedActionMetadata(
   const meta: NonNullable<ActionResult['execution_metadata']> = {}
   let i = 0
   while (i < buf.length) {
-    const [key, k] = readVarint(buf, i)
+    const [key, k] = readVarintAt(buf, i)
     i = k
     const field = key >>> 3
     const wire = key & 7
     if (wire === 2) {
-      const [len, l] = readVarint(buf, i)
+      const [len, l] = readVarintAt(buf, i)
       i = l
       const slice = buf.subarray(i, i + len)
       i += len
@@ -404,7 +413,7 @@ function decodeExecutedActionMetadata(
       else if (field === 7) meta.execution_start_timestamp = decodeTimestamp(slice)
       else if (field === 8) meta.execution_completed_timestamp = decodeTimestamp(slice)
     } else if (wire === 0) {
-      const [, n] = readVarint(buf, i)
+      const [, n] = readVarintAt(buf, i)
       i = n
     } else break
   }
@@ -416,10 +425,10 @@ function decodeTimestamp(buf: Uint8Array): { seconds?: string; nanos?: number }
   const ts: { seconds?: string; nanos?: number } = {}
   let i = 0
   while (i < buf.length) {
-    const [key, k] = readVarint(buf, i)
+    const [key, k] = readVarintAt(buf, i)
     i = k
     if ((key & 7) !== 0) break
-    const [v, n] = readVarint(buf, i)
+    const [v, n] = readVarintAt(buf, i)
     i = n
     if (key >>> 3 === 1) ts.seconds = String(v)
     else if (key >>> 3 === 2) ts.nanos = v
@@ -434,67 +443,26 @@ function decodeOutputDirectory(buf: Uint8Array): { path: string; tree_digest: Di
   const out = { path: '', tree_digest: { hash: '', size_bytes: 0 } }
   let i = 0
   while (i < buf.length) {
-    const [key, k] = readVarint(buf, i)
+    const [key, k] = readVarintAt(buf, i)
     i = k
     const field = key >>> 3
     const wire = key & 7
     if (wire === 0) {
-      const [, n] = readVarint(buf, i)
+      const [, n] = readVarintAt(buf, i)
       i = n
       continue
     }
     if (wire !== 2) break
-    const [len, l] = readVarint(buf, i)
+    const [len, l] = readVarintAt(buf, i)
     i = l
     const slice = buf.subarray(i, i + len)
     i += len
     if (field === 1) out.path = PATH_UTF8.decode(slice)
-    else if (field === 3) out.tree_digest = decodeDigest(slice)
+    else if (field === 3) out.tree_digest = decodeDigestBytes(slice)
   }
   return out
 }
 
-function decodeDigest(buf: Uint8Array): Digest {
-  const d: Digest = { hash: '', size_bytes: 0 }
-  let i = 0
-  while (i < buf.length) {
-    const [key, k] = readVarint(buf, i)
-    i = k
-    const field = key >>> 3
-    const wire = key & 7
-    if (wire === 2) {
-      const [len, l] = readVarint(buf, i)
-      i = l
-      if (field === 1) d.hash = new TextDecoder().decode(buf.subarray(i, i + len))
-      i += len
-    } else if (wire === 0) {
-      const [v, n] = readVarint(buf, i)
-      i = n
-      if (field === 2) d.size_bytes = v
-    } else break
-  }
-  return d
-}
-
-function readVarint(buf: Uint8Array, at: number): [number, number] {
-  let result = 0
-  let low = 0
-  let shift = 0
-  let i = at
-  for (;;) {
-    const byte = buf[i++]
-    if (byte === undefined) break
-    // Added, not OR-ed: a size of 4 GiB or more wrapped to 32 bits (F-43).
-    // Past 2^53 the value is a negative int32 sent as ten bytes (an exit
-    // code): its low 32 bits, which `| 0` reads back as the negative.
-    result += (byte & 0x7f) * 2 ** shift
-    if (shift < 32) low |= (byte & 0x7f) << shift
-    if ((byte & 0x80) === 0) break
-    shift += 7
-  }
-  return [result <= Number.MAX_SAFE_INTEGER ? result : low >>> 0, i]
-}
-
 /**
  * Backstop for an action the server never finishes. Generous on purpose: it
  * exists so a wedged remote cannot hang a run forever, not to police slow
@@ -1252,12 +1220,9 @@ async function this_readStream(
   return ''
 }
 
-/** Forwarded args are appended shell-quoted, exactly as the local executor does. */
+/** Forwarded args are placed by core's own `withForwardArgs`, as the local executor's are. */
 function fullCommand(req: ExecuteRequest, cdInto: string, projectRel: string): string {
-  const quoted =
-    req.forwardArgs.length === 0
-      ? req.command
-      : `${req.command} ${req.forwardArgs.map((a) => `'${a.replaceAll("'", `'\\''`)}'`).join(' ')}`
+  const command = withForwardArgs(req.command, req.forwardArgs)
   // A remote action gets NO PATH from this machine — sending one would put a
   // host path in the action digest and split every laptop from every runner.
   // But a task's command is normally a package binary (`oxlint`, `tsc`), and
@@ -1285,7 +1250,7 @@ function fullCommand(req: ExecuteRequest, cdInto: string, projectRel: string): s
   return (
     `VX_ROOT=${root}; ${cd}` +
     `export PATH="$VX_ROOT/node_modules/.bin:$PWD/node_modules/.bin:$PATH"; ` +
-    quoted
+    command
   )
 }
 

package/src/merkle.ts

@@ -828,7 +828,7 @@ function decodeSymlinkNode(buf: Uint8Array): SymlinkNode {
   return sl
 }
 
-function decodeDigestBytes(buf: Uint8Array): Digest {
+export function decodeDigestBytes(buf: Uint8Array): Digest {
   const d: Digest = { hash: '', size_bytes: 0 }
   let i = 0
   while (i < buf.length) {
@@ -850,7 +850,7 @@ function decodeDigestBytes(buf: Uint8Array): Digest {
   return d
 }
 
-function readVarintAt(buf: Uint8Array, at: number): [number, number] {
+export function readVarintAt(buf: Uint8Array, at: number): [number, number] {
   let result = 0
   let low = 0
   let shift = 0

package/src/wire.ts

@@ -1002,6 +1002,11 @@ export class ReapiClient {
     )
   }
 
+  /** The largest `Blob` `writeBlob` reads whole rather than streams. */
+  get wholeWriteBytes(): number {
+    return this.negotiatedBatchBytes || SAFE_BATCH_BYTES
+  }
+
   /**
    * Upload a blob via ByteStream, chunked at `chunkBytes`, RESUMING an
    * interrupted identity upload from the server's committed offset
@@ -1017,8 +1022,7 @@ export class ReapiClient {
    * zstd artifact already. A Blob under the limit is small enough to read.
    */
   async writeBlob(digest: Digest, source: Uint8Array | Blob): Promise<void> {
-    const streamed =
-      source instanceof Blob && source.size > (this.negotiatedBatchBytes || SAFE_BATCH_BYTES)
+    const streamed = source instanceof Blob && source.size > this.wholeWriteBytes
     const body = source instanceof Blob && !streamed ? await source.bytes() : source
     const compressed = this.compression && body instanceof Uint8Array
     // REAPI carries compression in the RESOURCE NAME: