@vzn/vx-reapi 0.0.652 → 0.0.653

package/src/index.ts

@@ -97,7 +97,7 @@ function connection(options: ReapiPluginOptions): ReapiOptions | undefined {
   // socket path is the file system's to read.
   const given = options.endpoint?.trim()
   const endpoint =
-    given?.startsWith('unix') === true ? given : given?.replace(/^(.*[^/:])\/+$/, '$1')
+    given?.startsWith('unix:') === true ? given : given?.replace(/^(.*[^/:])\/+$/, '$1')
   if (endpoint === undefined || endpoint === '') return undefined
   assertEndpoint(endpoint, from)
   // A secrets file's trailing newline sent a name no blob is under, and

package/src/merkle.ts

@@ -30,11 +30,11 @@ interface InputTree {
   /** File count, for logging/metrics. */
   fileCount: number
   /**
-   * Paths where a grafted subtree SHADOWED a disk-built directory of the
-   * same name. The graft wins (it is the upstream's authoritative output;
-   * the disk copy at best a stale materialisation) — but any DECLARED input
-   * files under that path are silently absent from the tree, so the caller
-   * should surface these.
+   * Paths where a grafted subtree SHADOWED declared input files read from
+   * disk under the same name. The graft wins (it is the upstream's
+   * authoritative output; the disk copy at best a stale materialisation) —
+   * but those files are absent from the tree, so the caller should surface
+   * these. Another upstream's output under a graft is merged in instead.
    */
   shadowed: string[]
   /**
@@ -102,7 +102,10 @@ export interface TreeGraft {
  * the graph self-consistent regardless of who produced it — file digests are
  * untouched (content-addressed, already in the CAS).
  */
-function canonicaliseTree(graft: TreeGraft): { root: Digest; blobs: Blob[] } {
+function canonicaliseTree(
+  graft: TreeGraft,
+  known: Map<string, Directory>,
+): { root: Digest; blobs: Blob[] } {
   // Keyed by the worker's own bytes first: a child re-encoded by US matches
   // its parent's reference only when both encoders agree byte for byte, and
   // one that does not (a field order, a `node_properties` our decoder drops)
@@ -136,6 +139,7 @@ function canonicaliseTree(graft: TreeGraft): { root: Digest; blobs: Blob[] } {
     const data = encodeDirectory(canonical)
     const digest = sha256(data)
     blobs.push({ digest, data })
+    known.set(digest.hash, canonical)
     return digest
   }
   return { root: rebuild(graft.root), blobs }
@@ -202,6 +206,61 @@ export async function buildInputTree(args: {
   const seen = new Set<string>()
   const moved: string[] = []
   let fileCount = 0
+  // A graft's directories by canonical digest, and the nodes opened from
+  // them: another upstream's output under a Tree graft is inserted into it,
+  // never dropped by it (prisma's client under install's node_modules).
+  const known = new Map<string, Directory>()
+  const fromGraft = new WeakSet<DirNode>()
+  const opened = (raw: Digest, at: string): Directory => {
+    const dir = known.get(raw.hash)
+    if (dir === undefined) {
+      throw new Error(
+        `vx-reapi: cannot add an upstream output under ${at}: the upstream Tree that holds it does not list it`,
+      )
+    }
+    return dir
+  }
+  const descend = (node: DirNode, seg: string, at: string): DirNode => {
+    let next = node.dirs.get(seg)
+    if (next !== undefined) return next
+    const raw = node.rawDirs.get(seg)
+    if (raw === undefined) {
+      next = emptyDir()
+    } else {
+      const dir = opened(raw, at)
+      next = {
+        files: new Map(dir.files.map((f) => [f.name, f])),
+        dirs: new Map(),
+        symlinks: new Map(dir.symlinks.map((s) => [s.name, s])),
+        rawDirs: new Map(dir.directories.map((d) => [d.name, d.digest])),
+      }
+      fromGraft.add(next)
+      node.rawDirs.delete(seg)
+    }
+    node.dirs.set(seg, next)
+    return next
+  }
+  const insertAt = (rel: string): { node: DirNode; leaf: string } => {
+    const parts = rel.split('/')
+    let node = root
+    for (let i = 0; i < parts.length - 1; i++) {
+      node = descend(node, parts[i]!, parts.slice(0, i + 1).join('/'))
+    }
+    return { node, leaf: parts[parts.length - 1]! }
+  }
+  /** A graft's directory laid over `target`, merging where both hold a directory. */
+  const overlay = (target: DirNode, raw: Digest, at: string): void => {
+    const dir = opened(raw, at)
+    for (const f of dir.files) target.files.set(f.name, f)
+    for (const s of dir.symlinks) target.symlinks.set(s.name, s)
+    for (const d of dir.directories) {
+      if (target.dirs.has(d.name) || target.rawDirs.has(d.name)) {
+        overlay(descend(target, d.name, `${at}/${d.name}`), d.digest, `${at}/${d.name}`)
+      } else {
+        target.rawDirs.set(d.name, d.digest)
+      }
+    }
+  }
 
   // Insertion order is NOT what makes the tree deterministic — `encodeDirectory`
   // sorts every node's names at encode time, and it has to, because grafts are
@@ -232,27 +291,27 @@ export async function buildInputTree(args: {
     if (!st.isFile()) return { kind: 'other' as const }
     return { kind: 'file' as const, data: await read(abs), mode: st.mode }
   })
+  // A declared input is placed before the Tree grafts, which replace one
+  // under them (`shadowed`); an upstream's output read from disk after
+  // them, as it is that upstream's own and lands beside theirs.
+  const later: (() => void)[] = []
   for (const [i, rel] of sorted.entries()) {
     const got = reads[i]!
     if (got.kind === 'gone') {
       moved.push(rel)
       continue
     }
+    const deferred = args.expected !== undefined && !args.expected.has(rel)
     if (got.kind === 'link') {
       const target = got.target
       const want = args.expected?.get(rel)
       if (want !== undefined && !carriesOid(new TextEncoder().encode(target), want)) moved.push(rel)
-      const parts = rel.split('/')
-      let node = root
-      for (const seg of parts.slice(0, -1)) {
-        let next = node.dirs.get(seg)
-        if (next === undefined) {
-          next = emptyDir()
-          node.dirs.set(seg, next)
-        }
-        node = next
+      const place = (): void => {
+        const { node, leaf } = insertAt(rel)
+        node.symlinks.set(leaf, { name: leaf, target })
       }
-      node.symlinks.set(parts[parts.length - 1]!, { name: parts[parts.length - 1]!, target })
+      if (deferred) later.push(place)
+      else place()
       continue
     }
     if (got.kind === 'other') {
@@ -272,40 +331,47 @@ export async function buildInputTree(args: {
       seen.add(digest.hash)
       blobs.push({ digest, data })
     }
-    const parts = rel.split('/')
-    let node = root
-    for (const seg of parts.slice(0, -1)) {
-      let next = node.dirs.get(seg)
-      if (next === undefined) {
-        next = emptyDir()
-        node.dirs.set(seg, next)
-      }
-      node = next
-    }
     // The owner-execute bit is what REAPI models; group/other add nothing a
     // worker can act on.
-    node.files.set(parts[parts.length - 1]!, {
-      name: parts[parts.length - 1]!,
-      digest,
-      is_executable: (got.mode & 0o100) !== 0,
-    })
+    const isExecutable = (got.mode & 0o100) !== 0
+    const place = (): void => {
+      const { node, leaf } = insertAt(rel)
+      node.files.set(leaf, { name: leaf, digest, is_executable: isExecutable })
+    }
+    if (deferred) later.push(place)
+    else place()
     fileCount++
   }
 
-  const insertAt = (rel: string): { node: DirNode; leaf: string } => {
-    const parts = rel.split('/')
-    let node = root
-    for (const seg of parts.slice(0, -1)) {
-      let next = node.dirs.get(seg)
-      if (next === undefined) {
-        next = emptyDir()
-        node.dirs.set(seg, next)
+  // Ancestors first, so a nested graft finds the one above it to merge into.
+  const shadowed: string[] = []
+  const treeGrafts = [...(args.treeGrafts ?? [])].sort((a, b) =>
+    a.path < b.path ? -1 : a.path > b.path ? 1 : 0,
+  )
+  for (const graft of treeGrafts) {
+    const { node, leaf } = insertAt(graft.path)
+    const canonical = canonicaliseTree(graft, known)
+    for (const b of canonical.blobs) {
+      if (!seen.has(b.digest.hash)) {
+        seen.add(b.digest.hash)
+        blobs.push(b)
       }
-      node = next
     }
-    return { node, leaf: parts[parts.length - 1]! }
+    const there = node.dirs.get(leaf)
+    if (node.rawDirs.has(leaf) || (there !== undefined && fromGraft.has(there))) {
+      overlay(descend(node, leaf, graft.path), canonical.root, graft.path)
+      continue
+    }
+    // The graft wins over declared inputs read from disk: it is the
+    // upstream's authoritative output, the disk copy at best a stale
+    // materialisation of it.
+    if (there !== undefined) {
+      shadowed.push(graft.path)
+      node.dirs.delete(leaf)
+    }
+    node.rawDirs.set(leaf, canonical.root)
   }
-
+  for (const place of later) place()
   for (const graft of args.fileGrafts ?? []) {
     const { node, leaf } = insertAt(graft.path)
     node.files.set(leaf, { name: leaf, digest: graft.digest, is_executable: graft.isExecutable })
@@ -317,31 +383,13 @@ export async function buildInputTree(args: {
   }
   for (const dir of args.ensureDirs ?? []) {
     if (dir === '') continue
+    const parts = dir.split('/')
     let node = root
-    for (const seg of dir.split('/')) {
-      let next = node.dirs.get(seg)
-      if (next === undefined) {
-        next = emptyDir()
-        node.dirs.set(seg, next)
-      }
-      node = next
+    for (let i = 0; i < parts.length; i++) {
+      node = descend(node, parts[i]!, parts.slice(0, i + 1).join('/'))
     }
   }
 
-  const shadowed: string[] = []
-  for (const graft of args.treeGrafts ?? []) {
-    const { node, leaf } = insertAt(graft.path)
-    if (node.dirs.has(leaf)) shadowed.push(graft.path)
-    const canonical = canonicaliseTree(graft)
-    for (const b of canonical.blobs) {
-      if (!seen.has(b.digest.hash)) {
-        seen.add(b.digest.hash)
-        blobs.push(b)
-      }
-    }
-    node.rawDirs.set(leaf, canonical.root)
-  }
-
   const rootDigest = serialise(root, blobs, seen)
   return { root: rootDigest, blobs, fileCount, shadowed, moved }
 }
@@ -384,13 +432,11 @@ function serialise(node: DirNode, blobs: Blob[], seen: Set<string>): Digest {
   const directories: DirectoryNode[] = []
   const dirNames = utf8Sorted([...new Set([...node.dirs.keys(), ...node.rawDirs.keys()])])
   for (const name of dirNames) {
+    // A name is in one map or the other, never both (`buildInputTree`).
     const sub = node.dirs.get(name)
     directories.push({
       name,
-      // A grafted subtree wins over a disk-built one of the same name: the
-      // graft is the upstream's AUTHORITATIVE output, the disk copy at best
-      // a stale materialisation of it.
-      digest: node.rawDirs.get(name) ?? serialise(sub!, blobs, seen),
+      digest: sub === undefined ? node.rawDirs.get(name)! : serialise(sub, blobs, seen),
     })
   }
   const files = utf8Sorted([...node.files.keys()]).map((n) => node.files.get(n)!)

package/src/wire.ts

@@ -318,6 +318,16 @@ function spent(err: unknown): unknown {
 
 const RETRY_DELAYS_MS = [100, 400, 1600]
 
+/**
+ * Re-attaching to an operation. A proxy that cuts long streams cuts a
+ * progressing action again and again, so only attaches that delivered no
+ * operation spend `RETRY_DELAYS_MS`'s budget. The wait still grows per
+ * consecutive cut, so a server that answers EXECUTING and hangs up is not
+ * re-dialled every 100 ms; it starts over once a stream advanced the stage
+ * or stayed open `steadyMs`. A mutable object: tests shorten it.
+ */
+export const reattach = { backoffMs: [100, 400, 1600, 5000], steadyMs: 10_000 }
+
 const deadlineOf = (bounds: grpc.CallOptions): number => (bounds.deadline as Date).getTime()
 
 const executionAborted = (): Error => new Error('execution aborted')
@@ -909,17 +919,19 @@ export class ReapiClient {
   /**
    * Upload blobs, choosing the transport per blob: batched while they fit the
    * server's batch budget, ByteStream for the rest. Only what
-   * `FindMissingBlobs` reports absent is sent at all.
+   * `FindMissingBlobs` reports absent is sent, unless `everyBlob`: a server
+   * whose Execute just named them missing has already contradicted it.
    */
   async uploadBlobs(
     blobs: ReadonlyArray<{ digest: Digest; data: Uint8Array }>,
     maxBatchBytes = 0,
+    everyBlob = false,
   ): Promise<void> {
     if (blobs.length === 0) return
-    const missing = new Set(
-      (await this.findMissingBlobs(blobs.map((b) => b.digest))).map((d) => d.hash),
-    )
-    const todo = blobs.filter((b) => missing.has(b.digest.hash))
+    const missing = everyBlob
+      ? undefined
+      : new Set((await this.findMissingBlobs(blobs.map((b) => b.digest))).map((d) => d.hash))
+    const todo = missing === undefined ? blobs : blobs.filter((b) => missing.has(b.digest.hash))
     // 0 means the server did not say; the gRPC default max message is 4 MB and
     // the request carries framing on top, so leave headroom rather than
     // discovering the limit as a RESOURCE_EXHAUSTED mid-run.
@@ -1009,14 +1021,14 @@ export class ReapiClient {
 
   /**
    * `GetActionResult`; `null` on NOT_FOUND — a miss is not an error.
-   * `inline` asks the server to return stdout and the named output files in
-   * the reply. The spec lets it decline (and requires it past the message
-   * limit), so a caller reads `stdout_raw` / `contents` when present and
-   * fetches otherwise.
+   * `inline` asks the server to return the streams it names and the named
+   * output files in the reply. The spec lets it decline (and requires it
+   * past the message limit), so a caller reads `*_raw` / `contents` when
+   * present and fetches otherwise.
    */
   async getActionResult(
     action: Digest,
-    inline?: { stdout: boolean; files: readonly string[] },
+    inline?: { stdout: boolean; stderr?: boolean; files: readonly string[] },
   ): Promise<ActionResult | null> {
     try {
       const result = await this.unary<ActionResult>(
@@ -1027,7 +1039,11 @@ export class ReapiClient {
           action_digest: action,
           ...(inline === undefined
             ? {}
-            : { inline_stdout: inline.stdout, inline_output_files: inline.files }),
+            : {
+                inline_stdout: inline.stdout,
+                ...(inline.stderr === true ? { inline_stderr: true } : {}),
+                inline_output_files: inline.files,
+              }),
         },
         this.meta(),
         this.boundedMeta(),
@@ -1317,9 +1333,13 @@ export class ReapiClient {
    * the bytes pass and a mismatch errors the stream at its end, so no reader
    * reaches the end of a blob that is not the one asked for.
    * Identity-encoded, as `writeBlob` streams: the blob this reads is a zstd
-   * artifact already.
+   * artifact already. `refused` is what that error becomes, for a reader
+   * that tells wrong bytes from a cut.
    */
-  async readBlobStream(digest: Digest): Promise<ReadableStream<Uint8Array> | null> {
+  async readBlobStream(
+    digest: Digest,
+    refused: (err: Error) => Error = (err) => err,
+  ): Promise<ReadableStream<Uint8Array> | null> {
     const resource = `${this.instance ? `${this.instance}/` : ''}blobs/${digest.hash}/${digest.size_bytes}`
     type Message = { data: Uint8Array }
     let call: AsyncIterable<Message> & { cancel(): void }
@@ -1366,14 +1386,18 @@ export class ReapiClient {
         const got = next ?? (await take())
         next = undefined
         if (got.done) {
-          assertServed(size, hasher && (() => hasher.digest('hex')), digest)
+          try {
+            assertServed(size, hasher && (() => hasher.digest('hex')), digest)
+          } catch (err) {
+            throw refused(err as Error)
+          }
           controller.close()
           return
         }
         size += got.value.data.length
         if (size > Number(digest.size_bytes)) {
           call.cancel()
-          throw overServed(digest)
+          throw refused(overServed(digest))
         }
         hasher?.update(got.value.data)
         controller.enqueue(got.value.data)
@@ -1421,27 +1445,47 @@ export class ReapiClient {
       ...(this.digestFunction === 'SHA256' ? {} : { digest_function: this.digestFunction }),
     }
     let operationName = ''
-    for (let attempt = 0; ; attempt++) {
+    // Attaches in a row that delivered no operation, cuts since a stream last
+    // advanced or held, and the furthest stage reached.
+    let empty = 0
+    let cuts = 0
+    let reached = -1
+    for (;;) {
       let failure: unknown
+      let delivered = false
+      let advanced = false
+      const opened = performance.now()
+      const onStage = (stage: string): void => {
+        const rank = EXEC_STAGE.indexOf(stage as (typeof EXEC_STAGE)[number])
+        if (rank > reached) {
+          reached = rank
+          advanced = true
+        }
+        opts.onStage?.(stage)
+      }
       try {
         const op =
           operationName === ''
-            ? await this.operationStream('execute', req, signal, opts.onStage, (n) => {
-                operationName = n
-                opts.onOperation?.(n)
+            ? await this.operationStream('execute', req, signal, onStage, (o) => {
+                delivered = true
+                if (o.name === '' || o.name === undefined) return
+                operationName = o.name
+                opts.onOperation?.(o.name)
               })
             : await this.operationStream(
                 'waitExecution',
                 { name: operationName },
                 signal,
-                opts.onStage,
+                onStage,
+                () => {
+                  delivered = true
+                },
               )
         if (op.done) return op
         // A stream can END cleanly on an operation still QUEUED or EXECUTING
         // (a server or proxy cutting long streams); the operation lives on,
-        // so it is re-attached as a dropped stream is, on the same budget.
-        // An unnamed one cannot be re-attached, and a plain Error is not
-        // retried.
+        // so it is re-attached as a dropped stream is. An unnamed one cannot
+        // be re-attached, and a plain Error is not retried.
         if (operationName === '') {
           throw new Error('execution stream closed before the operation finished')
         }
@@ -1454,12 +1498,18 @@ export class ReapiClient {
         // (a restart lost it). Nothing is running to re-attach to, and the
         // action is the same bytes, so it is executed again, on the same
         // budget — as Bazel's executor does.
-        if (code === NOT_FOUND && operationName !== '') operationName = ''
-        else if (!isRetryable(code)) throw err
+        if (code === NOT_FOUND && operationName !== '') {
+          operationName = ''
+          reached = -1
+        } else if (!isRetryable(code)) throw err
         failure = err
       }
-      const delay = RETRY_DELAYS_MS[attempt]
-      if (delay === undefined) throw failure
+      // An EXECUTING-then-cut loop never spends this budget; the executor's
+      // stall bound (`exec.timeout` / `executeTimeoutMs`) ends it.
+      empty = delivered ? 0 : empty + 1
+      if (empty > RETRY_DELAYS_MS.length) throw failure
+      if (advanced || performance.now() - opened >= reattach.steadyMs) cuts = 0
+      const delay = reattach.backoffMs[Math.min(cuts++, reattach.backoffMs.length - 1)]!
       await abortableSleep(delay, signal)
     }
   }
@@ -1495,7 +1545,7 @@ export class ReapiClient {
     req: unknown,
     signal?: AbortSignal,
     onStage?: (stage: string) => void,
-    onName?: (name: string) => void,
+    onOperation?: (op: Operation) => void,
   ): Promise<Operation> {
     return new Promise((resolve, reject) => {
       // An aborted signal never fires `abort` again, and the stream has no
@@ -1514,7 +1564,7 @@ export class ReapiClient {
       signal?.addEventListener('abort', onAbort, { once: true })
       stream.on('data', (op: Operation) => {
         last = op
-        if (op.name !== '' && op.name !== undefined && onName !== undefined) onName(op.name)
+        onOperation?.(op)
         // ExecuteOperationMetadata carries the action's STAGE
         // (QUEUED / EXECUTING / COMPLETED). Surfacing it is the difference
         // between "vx is hung" and "the action is queued behind 40 others".