@vzn/vx-reapi 0.0.646 → 0.0.648

package/src/index.ts

@@ -23,7 +23,7 @@ import {
 } from '@vzn/vx'
 import { ReapiRemoteCache } from './cache.js'
 import { reapiExecutor } from './executor.js'
-import { ReapiClient, type ReapiOptions, usesTls } from './wire.js'
+import { MAX_TIMER_MS, ReapiClient, type ReapiOptions, usesTls } from './wire.js'
 
 // `ReapiRemoteCache` is public for a workspace that composes cache
 // layers by hand; the wire, the Merkle encoders and the executor are
@@ -50,8 +50,8 @@ export interface ReapiPluginOptions extends Partial<
    */
   queueTimeoutMs?: number
   /**
-   * Endpoint, or omit to read `VX_REAPI_ENDPOINT`. With neither the plugin
-   * DECLINES — a declared-but-unconfigured plugin costs nothing and must
+   * Endpoint (`process.env.REAPI_ENDPOINT` in `.vx/workspace.ts`). Unset
+   * or empty, the plugin DECLINES — a declared-but-unconfigured plugin costs nothing and must
    * never fail a run. TLS unless it starts `grpc://`, `http://`, `unix:` or
    * `unix-abstract:`, or `tls: false` is set, as Bazel reads a schemeless
    * `--remote_cache`.
@@ -61,8 +61,7 @@ export interface ReapiPluginOptions extends Partial<
    * Contribute the `executor` capability too, so tasks RUN on the REAPI
    * server rather than only caching there. Default false: remote execution
    * changes where a user's build runs, which is not something a plugin should
-   * switch on merely by being configured for caching. `VX_REAPI_EXECUTE=1`
-   * also enables it.
+   * switch on merely by being configured for caching.
    */
   execute?: boolean
   /** REAPI platform properties for remote execution (`container-image`, …). */
@@ -71,14 +70,12 @@ export interface ReapiPluginOptions extends Partial<
   capacity?: number
   /**
    * PEM file of the CA that signed the server's certificate, for a server
-   * behind a private CA. Falls back to `VX_REAPI_TLS_CERTIFICATE`. Bazel's
-   * `--tls_certificate`.
+   * behind a private CA. Bazel's `--tls_certificate`.
    */
   tlsCertificate?: string
   /**
    * PEM files of a client certificate and its key, for a server that asks
-   * for mutual TLS (`VX_REAPI_TLS_CLIENT_CERTIFICATE` / `_KEY`). Bazel's
-   * `--tls_client_certificate` / `--tls_client_key`.
+   * for mutual TLS. Bazel's `--tls_client_certificate` / `--tls_client_key`.
    */
   tlsClientCertificate?: string
   tlsClientKey?: string
@@ -95,69 +92,51 @@ export interface ReapiPluginOptions extends Partial<
  * Declines when no endpoint is configured, so it is safe to leave declared.
  */
 function connection(options: ReapiPluginOptions): ReapiOptions | undefined {
-  // `process.env`, not `Bun.env`, for core's reason (exec/sandbox-runtime.ts):
-  // an embedder that replaces the env object leaves `Bun.env` on the old one.
-  const from = options.endpoint !== undefined ? '`reapi({ endpoint })`' : 'VX_REAPI_ENDPOINT'
-  const endpoint = (options.endpoint ?? process.env['VX_REAPI_ENDPOINT'])?.trim()
+  const from = '`reapi({ endpoint })`'
+  // A trailing slash is stripped, as every other plugin strips its URL's; a
+  // socket path is the file system's to read.
+  const given = options.endpoint?.trim()
+  const endpoint =
+    given?.startsWith('unix') === true ? given : given?.replace(/^(.*[^/:])\/+$/, '$1')
   if (endpoint === undefined || endpoint === '') return undefined
   assertEndpoint(endpoint, from)
   // A secrets file's trailing newline sent a name no blob is under, and
   // every lookup missed in silence.
-  const instanceName =
-    (options.instanceName ?? process.env['VX_REAPI_INSTANCE'])?.trim() || undefined
+  const instanceName = options.instanceName?.trim() || undefined
   // A server behind a private CA, or one that asks for a client
   // certificate, was unreachable: TLS used the system roots and no client
   // pair (F-41). Each is a PEM file, read here; one that cannot be read is
   // a setting to fix, refused naming it.
-  const pem = (
-    option: string | undefined,
-    name: string,
-    fromEnv: string | undefined,
-    env: string,
-  ): string | undefined => {
-    const file = option ?? fromEnv?.trim()
-    if (file === undefined || file === '') return undefined
+  const pem = (file: string | undefined, name: string): string | undefined => {
+    if (file === undefined || file.trim() === '') return undefined
     try {
-      return readFileSync(file, 'utf8')
+      return readFileSync(file.trim(), 'utf8')
     } catch (err) {
-      const from = option !== undefined ? `\`reapi({ ${name} })\`` : env
       throw new UserError(
-        `vx/reapi: ${from} names ${file}, which cannot be read (${(err as NodeJS.ErrnoException).code ?? String(err)})`,
+        `\`reapi({ ${name} })\` names ${file.trim()}, which cannot be read (${(err as NodeJS.ErrnoException).code ?? String(err)})`,
       )
     }
   }
-  const tlsCaPem = pem(
-    options.tlsCertificate,
-    'tlsCertificate',
-    process.env['VX_REAPI_TLS_CERTIFICATE'],
-    'VX_REAPI_TLS_CERTIFICATE',
-  )
-  const tlsClientCertPem = pem(
-    options.tlsClientCertificate,
-    'tlsClientCertificate',
-    process.env['VX_REAPI_TLS_CLIENT_CERTIFICATE'],
-    'VX_REAPI_TLS_CLIENT_CERTIFICATE',
-  )
-  const tlsClientKeyPem = pem(
-    options.tlsClientKey,
-    'tlsClientKey',
-    process.env['VX_REAPI_TLS_CLIENT_KEY'],
-    'VX_REAPI_TLS_CLIENT_KEY',
-  )
+  const tlsCaPem = pem(options.tlsCertificate, 'tlsCertificate')
+  const tlsClientCertPem = pem(options.tlsClientCertificate, 'tlsClientCertificate')
+  const tlsClientKeyPem = pem(options.tlsClientKey, 'tlsClientKey')
   // grpc-js refuses either one alone, with a message that names neither
   // setting.
   if ((tlsClientCertPem === undefined) !== (tlsClientKeyPem === undefined)) {
     throw new UserError(
-      'vx/reapi: a client certificate and its key go together — set both tlsClientCertificate and tlsClientKey (VX_REAPI_TLS_CLIENT_CERTIFICATE / VX_REAPI_TLS_CLIENT_KEY), or neither',
+      'reapi(): a client certificate and its key go together — set both tlsClientCertificate and tlsClientKey, or neither',
     )
   }
   if (usesTls({ endpoint, tls: options.tls, tlsCaPem, tlsClientCertPem }) && ipTarget(endpoint)) {
     throw new UserError(
-      `vx/reapi: ${from} is ${JSON.stringify(endpoint)}, TLS to an IP address, which Bun cannot open (it refuses an IP as the TLS server name) — name the server by host name, or use grpc:// for plaintext`,
+      `${from} is ${JSON.stringify(endpoint)}, TLS to an IP address, which Bun cannot open (it refuses an IP as the TLS server name) — name the server by host name, or use grpc:// for plaintext`,
     )
   }
+  // Never the option as written: a blank one spread here was sent as the
+  // instance name.
+  const { instanceName: _written, ...rest } = options
   return {
-    ...options,
+    ...rest,
     endpoint,
     ...(instanceName === undefined ? {} : { instanceName }),
     ...(tlsCaPem === undefined ? {} : { tlsCaPem }),
@@ -181,7 +160,7 @@ function assertEndpoint(endpoint: string, from: string): void {
   const port = m?.[2] === undefined ? undefined : Number(m[2])
   if (m === null || (port !== undefined && (port < 1 || port > 65_535))) {
     throw new UserError(
-      `vx/reapi: ${from} is ${JSON.stringify(endpoint)}, which is not host[:port] (e.g. grpcs://cache.example.com:443)`,
+      `${from} is ${JSON.stringify(endpoint)}, which is not host[:port] (e.g. grpcs://cache.example.com:443)`,
     )
   }
 }
@@ -222,15 +201,49 @@ const REAPI_PLUGIN_KEYS: PluginOptionKinds<ReapiPluginOptions> = {
   chunkBytes: 'number',
 }
 
+/**
+ * The number options, checked at the call: the client and the executor
+ * refused a bad one only once an endpoint was set, so a laptop passed what
+ * failed the first CI run, and a NaN read as `null`.
+ */
+function refuseBadNumbers(options: ReapiPluginOptions): void {
+  const refuse = (name: string, v: unknown, want: string): never => {
+    throw new UserError(`reapi() option "${name}" must be ${want}, got ${String(v)}`)
+  }
+  for (const name of [
+    'callTimeoutMs',
+    'metaTimeoutMs',
+    'executeTimeoutMs',
+    'queueTimeoutMs',
+  ] as const) {
+    const v = options[name]
+    if (v !== undefined && !(v > 0 && v <= MAX_TIMER_MS))
+      refuse(name, v, `a positive number of ms up to ${MAX_TIMER_MS}`)
+  }
+  for (const name of ['chunkBytes', 'capacity'] as const) {
+    const v = options[name]
+    if (v !== undefined && !(Number.isInteger(v) && v > 0))
+      refuse(name, v, 'a whole number above 0')
+  }
+}
+
 export function reapi(options: ReapiPluginOptions = {}): VxPlugin {
   refuseUnknownOptions('reapi()', options, REAPI_PLUGIN_KEYS)
+  refuseBadNumbers(options)
+  // An unset variable as a value was sent as an empty one on every action.
+  for (const key of ['platform', 'headers'] as const)
+    for (const [name, value] of Object.entries(options[key] ?? {}))
+      if (typeof value !== 'string')
+        throw new UserError(
+          `reapi() option "${key}.${name}" must be a string, got ${key === 'headers' && value !== undefined ? typeof value : String(value)}`,
+        )
   let executorClient: ReapiClient | undefined
   let remoteCache: ReapiRemoteCache | undefined
   return definePlugin(import.meta, {
     async executor(ctx): Promise<TaskExecutor | undefined> {
       // Before the connection: with execute off, an unreadable PEM is the
       // cache hook's to refuse, never a failure "in executor".
-      if (options.execute !== true && process.env['VX_REAPI_EXECUTE'] !== '1') return undefined
+      if (options.execute !== true) return undefined
       const conn = connection(options)
       if (conn === undefined) return undefined
       executorClient = new ReapiClient({ ...conn, onWarn: (m) => ctx.warn(m) })
@@ -241,8 +254,12 @@ export function reapi(options: ReapiPluginOptions = {}): VxPlugin {
       // plugin's setting nor anything to do about it. Core turns a throwing
       // factory into a UserError and ABORTS — deliberately, an executor is
       // load-bearing — so this is the message a user acts on.
+      // Both asks: a failed second one escaped as the raw status, naming
+      // neither the server nor the setting.
+      let caps: Awaited<ReturnType<ReapiClient['capabilities']>>
       try {
         await executorClient.negotiate()
+        caps = await executorClient.capabilities()
       } catch (err) {
         executorClient.close()
         executorClient = undefined
@@ -262,10 +279,9 @@ export function reapi(options: ReapiPluginOptions = {}): VxPlugin {
       // A cache-only deployment (bazel-remote) advertises no execution
       // capability. Offering it work would hang the run on a server that will
       // never answer, so DECLINE loudly and let the local executor take over.
-      const caps = await executorClient.capabilities()
       if (!caps.execEnabled) {
         ctx.warn(
-          `vx/reapi: ${conn.endpoint} does not advertise remote execution (cache only) — tasks will run locally`,
+          `[vx] vx-reapi: ${conn.endpoint} does not advertise remote execution (cache only) — tasks will run locally`,
         )
         executorClient.close()
         executorClient = undefined

package/src/merkle.ts

@@ -115,13 +115,22 @@ function canonicaliseTree(graft: TreeGraft): { root: Digest; blobs: Blob[] } {
     if (sent !== undefined) byOldDigest.set(sent, child)
   })
   const blobs: Blob[] = []
+  // Once per old digest: a subtree the Tree names under many names was
+  // rebuilt per name, 2^depth times.
+  const rebuilt = new Map<string, Digest>()
   const rebuild = (dir: Directory): Digest => {
     const directories: DirectoryNode[] = dir.directories.map((d) => {
       const child = byOldDigest.get(d.digest.hash)
       // A child we cannot resolve keeps its original digest — the blob may
       // exist server-side under the worker's encoding even if we cannot
       // re-derive it. Better a possibly-dangling reference than a wrong one.
-      return child === undefined ? d : { name: d.name, digest: rebuild(child) }
+      if (child === undefined) return d
+      let digest = rebuilt.get(d.digest.hash)
+      if (digest === undefined) {
+        digest = rebuild(child)
+        rebuilt.set(d.digest.hash, digest)
+      }
+      return { name: d.name, digest }
     })
     const canonical: Directory = { files: dir.files, directories, symlinks: dir.symlinks }
     const data = encodeDirectory(canonical)
@@ -200,7 +209,9 @@ export async function buildInputTree(args: {
   // anyway so a warning naming several paths reads the same run to run.
   // Read READ_CONCURRENCY at once, inserted in sorted order below: one at a
   // time, 2 000 small inputs cost 304 ms of file-system round trips (F-36).
-  const sorted = [...args.paths].sort()
+  // Deduped: an upstream's output on disk that is also a declared input
+  // is named twice, and was read and counted twice.
+  const sorted = [...new Set(args.paths)].sort()
   const reads = await mapBounded(sorted, READ_CONCURRENCY, async (rel) => {
     const abs = path.join(args.workspaceRoot, rel)
     // lstat, not stat: a symlinked input must be REPRESENTED as a symlink.
@@ -244,7 +255,11 @@ export async function buildInputTree(args: {
       node.symlinks.set(parts[parts.length - 1]!, { name: parts[parts.length - 1]!, target })
       continue
     }
-    if (got.kind === 'other') continue
+    if (got.kind === 'other') {
+      // A keyed file that is now a directory (or a socket) is absent too.
+      if (args.expected?.has(rel) === true) moved.push(rel)
+      continue
+    }
     const data = got.data
     const want = args.expected?.get(rel)
     if (want !== undefined && !carriesOid(data, want)) moved.push(rel)
@@ -331,6 +346,33 @@ export async function buildInputTree(args: {
   return { root: rootDigest, blobs, fileCount, shadowed, moved }
 }
 
+/**
+ * REAPI's sort: names by UTF-8 bytes, which servers compare (Go's string
+ * order). JS's default sort is by UTF-16 units, which puts an astral name
+ * (an emoji) before one in U+E000-U+FFFF; a server refused the Directory,
+ * and another client hashed the same tree to another digest.
+ */
+function byUtf8(a: string, b: string): number {
+  const n = Math.min(a.length, b.length)
+  for (let i = 0; i < n; i++) {
+    let x = a.charCodeAt(i)
+    let y = b.charCodeAt(i)
+    if (x === y) continue
+    // Surrogates (D800-DFFF) are code points past FFFF: move them after E000-FFFF.
+    if (x >= 0xd800 && y >= 0xd800) {
+      x += x >= 0xe000 ? -0x800 : 0x2000
+      y += y >= 0xe000 ? -0x800 : 0x2000
+    }
+    return x - y
+  }
+  return a.length - b.length
+}
+
+/** `xs` sorted by `byUtf8`; the default sort when no unit can differ. */
+function utf8Sorted(xs: string[]): string[] {
+  return xs.some((x) => /[\ud800-\uffff]/.test(x)) ? xs.sort(byUtf8) : xs.sort()
+}
+
 /**
  * Depth-first, children before parents: a Directory's digest covers its
  * children's digests, so they must be finalised first. REAPI requires `files`
@@ -340,7 +382,7 @@ export async function buildInputTree(args: {
  */
 function serialise(node: DirNode, blobs: Blob[], seen: Set<string>): Digest {
   const directories: DirectoryNode[] = []
-  const dirNames = [...new Set([...node.dirs.keys(), ...node.rawDirs.keys()])].sort()
+  const dirNames = utf8Sorted([...new Set([...node.dirs.keys(), ...node.rawDirs.keys()])])
   for (const name of dirNames) {
     const sub = node.dirs.get(name)
     directories.push({
@@ -351,8 +393,8 @@ function serialise(node: DirNode, blobs: Blob[], seen: Set<string>): Digest {
       digest: node.rawDirs.get(name) ?? serialise(sub!, blobs, seen),
     })
   }
-  const files = [...node.files.keys()].sort().map((n) => node.files.get(n)!)
-  const symlinks = [...node.symlinks.keys()].sort().map((n) => node.symlinks.get(n)!)
+  const files = utf8Sorted([...node.files.keys()]).map((n) => node.files.get(n)!)
+  const symlinks = utf8Sorted([...node.symlinks.keys()]).map((n) => node.symlinks.get(n)!)
   const dir: Directory = { files, directories, symlinks }
   const data = encodeDirectory(dir)
   const digest = sha256(data)
@@ -517,7 +559,7 @@ const HASH_ALGO: Partial<Record<DigestFunctionName, string>> = {
  */
 export function digestWith(fn: DigestFunctionName, data: Uint8Array): Digest {
   const algo = HASH_ALGO[fn]
-  if (algo === undefined) throw new Error(`@vzn/vx-reapi: unsupported digest function ${fn}`)
+  if (algo === undefined) throw new Error(`unsupported digest function ${fn}`)
   return { hash: createHash(algo).update(data).digest('hex'), size_bytes: data.length }
 }
 
@@ -533,6 +575,7 @@ export function canDigest(fn: DigestFunctionName): boolean {
   try {
     createHash(algo)
     return true
+    // quiet: a hash this runtime lacks is answered by false
   } catch {
     return false
   }
@@ -584,15 +627,13 @@ interface CommandSpec {
  * over these bytes and canonical encoders write ascending field numbers.
  */
 export function encodeCommand(c: CommandSpec): Uint8Array {
-  const env = [...c.environmentVariables].sort((a, b) =>
-    a.name < b.name ? -1 : a.name > b.name ? 1 : 0,
-  )
-  const platform = [...c.platform].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))
+  const env = [...c.environmentVariables].sort((a, b) => byUtf8(a.name, b.name))
+  const platform = [...c.platform].sort((a, b) => byUtf8(a.name, b.name))
   return concat([
     ...c.arguments.map((a) => repStrField(1, a)),
     ...env.map((e) => lenField(2, concat([strField(1, e.name), strField(2, e.value)]))),
-    ...[...(c.legacyOutputFiles ?? [])].sort().map((p) => repStrField(3, p)),
-    ...[...(c.legacyOutputDirectories ?? [])].sort().map((p) => repStrField(4, p)),
+    ...utf8Sorted([...(c.legacyOutputFiles ?? [])]).map((p) => repStrField(3, p)),
+    ...utf8Sorted([...(c.legacyOutputDirectories ?? [])]).map((p) => repStrField(4, p)),
     ...(platform.length > 0
       ? [
           lenField(
@@ -604,8 +645,8 @@ export function encodeCommand(c: CommandSpec): Uint8Array {
         ]
       : []),
     ...(c.workingDirectory === '' ? [] : [strField(6, c.workingDirectory)]),
-    ...[...c.outputPaths].sort().map((p) => repStrField(7, p)),
-    ...[...(c.outputNodeProperties ?? [])].sort().map((n) => repStrField(8, n)),
+    ...utf8Sorted([...c.outputPaths]).map((p) => repStrField(7, p)),
+    ...utf8Sorted([...(c.outputNodeProperties ?? [])]).map((n) => repStrField(8, n)),
     ...(c.outputDirectoryFormat === undefined || c.outputDirectoryFormat === 0
       ? []
       : [intField(9, c.outputDirectoryFormat)]),
@@ -627,9 +668,7 @@ export function encodeAction(a: {
   /** `platform` (field 10) — REAPI v2.2 moved it here from `Command`. */
   platform?: ReadonlyArray<{ name: string; value: string }>
 }): Uint8Array {
-  const platform = [...(a.platform ?? [])].sort((x, y) =>
-    x.name < y.name ? -1 : x.name > y.name ? 1 : 0,
-  )
+  const platform = [...(a.platform ?? [])].sort((x, y) => byUtf8(x.name, y.name))
   return concat([
     lenField(1, encodeDigest(a.commandDigest)),
     lenField(2, encodeDigest(a.inputRootDigest)),