@visulima/task-runner 1.0.0-alpha.8 → 1.0.0-alpha.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/CHANGELOG.md +31 -0
  2. package/README.md +3 -1
  3. package/dist/index.d.ts +791 -205
  4. package/dist/index.js +28 -20
  5. package/dist/packem_chunks/index.js +5593 -0
  6. package/dist/packem_shared/{Cache-CWaX_c8U.js → Cache-CbhoA268.js} +151 -10
  7. package/dist/packem_shared/{FileAccessTracker-CQ5Ot7Hd.js → FileAccessTracker-D8zIURPU.js} +1 -1
  8. package/dist/packem_shared/{FingerprintManager-CV7U4f4f.js → FingerprintManager-78DjwWQ4.js} +1 -1
  9. package/dist/packem_shared/HttpRemoteCache-BTXUBH7t.js +290 -0
  10. package/dist/packem_shared/INPUT_URI_SCHEMES-DRm76YI5.js +69 -0
  11. package/dist/packem_shared/{IncrementalFileHasher-BRS76-mb.js → IncrementalFileHasher-BBhVK491.js} +1 -1
  12. package/dist/packem_shared/ReapiRemoteCache-vgRxDMmu.js +1012 -0
  13. package/dist/packem_shared/{TaskOrchestrator-UCMHCx8c.js → TaskOrchestrator-CdRaQhTO.js} +34 -11
  14. package/dist/packem_shared/{TrackedTaskExecutor-CFPpQfXF.js → TrackedTaskExecutor-CWSMfHAW.js} +2 -2
  15. package/dist/packem_shared/V2_ROOT-DKBLxKo4.js +14 -0
  16. package/dist/packem_shared/actionDigestForTaskHash-BRE-9MT6.js +121 -0
  17. package/dist/packem_shared/archive-CnggHWb-.js +152 -0
  18. package/dist/packem_shared/{collectFiles-ClXHnHhg.js → collectFiles-cc1gokGU.js} +2 -1
  19. package/dist/packem_shared/{computeTaskHash-B5APHW7e.js → computeTaskHash-DHoBJ_-V.js} +10 -4
  20. package/dist/packem_shared/containsBlob-CwGB0a_q.js +125 -0
  21. package/dist/packem_shared/{createTaskGraph-B5YrfAMx.js → createTaskGraph-Bwl4hwAf.js} +17 -0
  22. package/dist/packem_shared/{defaultTaskRunner-DzR0ld8F.js → defaultTaskRunner-BaX4ZbFv.js} +24 -13
  23. package/dist/packem_shared/{detectFrameworks-CeFzKE6J.js → detectFrameworks-D7nyTc-o.js} +1 -1
  24. package/dist/packem_shared/{detectScriptShell-CR-xXKA4.js → detectScriptShell-CzxCM9-t.js} +1 -1
  25. package/dist/packem_shared/digestBuffer-CPdI2E1d.js +48 -0
  26. package/dist/packem_shared/{expandArguments-0AwD2BIA.js → expandArguments-Ba-hHYff.js} +2 -1
  27. package/dist/packem_shared/{expandTokensInString-C03AGAjh.js → expandTokensInString-Bb7nYehP.js} +2 -1
  28. package/dist/packem_shared/{extractPackageName-CbVNW-dr.js → extractPackageName-CMHjqGj_.js} +1 -1
  29. package/dist/packem_shared/{generateRunSummary-BE1jnQ3H.js → generateRunSummary-Bah7CFay.js} +1 -1
  30. package/dist/packem_shared/{getCurrentBranch-DsKPDoVj.js → getCurrentBranch-DVNikt0P.js} +11 -8
  31. package/dist/packem_shared/getMainWorktreeRoot-iBqToQJ4.js +114 -0
  32. package/dist/packem_shared/{parseCommands-CJ16ohOB.js → parseCommands-DDdIxaH5.js} +3 -3
  33. package/dist/packem_shared/resolveCacheMode-CsmHT_0o.js +21 -0
  34. package/dist/packem_shared/{runConcurrently-CmfC4r-f.js → runConcurrently-BCGQ9fJl.js} +1 -1
  35. package/dist/packem_shared/shell-quote-DWJJbt21.js +3 -0
  36. package/dist/packem_shared/{utils-zO0ZRgtf.js → utils-Bmnj-H2J.js} +4 -1
  37. package/index.js +52 -52
  38. package/package.json +23 -10
  39. package/dist/packem_shared/RemoteCache-DSU3lc87.js +0 -219
  40. package/dist/packem_shared/archive-UQHAnZUa.js +0 -102
@@ -0,0 +1,1012 @@
1
+ import { createRequire as __cjs_createRequire } from "node:module";
2
+
3
+ const __cjs_require = __cjs_createRequire(import.meta.url);
4
+
5
+ const __cjs_getProcess = typeof globalThis !== "undefined" && typeof globalThis.process !== "undefined" ? globalThis.process : process;
6
+
7
+ const __cjs_getBuiltinModule = (module) => {
8
+ // Check if we're in Node.js and version supports getBuiltinModule
9
+ if (typeof __cjs_getProcess !== "undefined" && __cjs_getProcess.versions && __cjs_getProcess.versions.node) {
10
+ const [major, minor] = __cjs_getProcess.versions.node.split(".").map(Number);
11
+ // Node.js 20.16.0+ and 22.3.0+
12
+ if (major > 22 || (major === 22 && minor >= 3) || (major === 20 && minor >= 16)) {
13
+ return __cjs_getProcess.getBuiltinModule(module);
14
+ }
15
+ }
16
+ // Fallback to createRequire
17
+ return __cjs_require(module);
18
+ };
19
+
20
+ const {
21
+ randomUUID
22
+ } = __cjs_getBuiltinModule("node:crypto");
23
+ const {
24
+ createWriteStream
25
+ } = __cjs_getBuiltinModule("node:fs");
26
+ const {
27
+ mkdir,
28
+ rm
29
+ } = __cjs_getBuiltinModule("node:fs/promises");
30
+ const {
31
+ Readable
32
+ } = __cjs_getBuiltinModule("node:stream");
33
+ const {
34
+ pipeline
35
+ } = __cjs_getBuiltinModule("node:stream/promises");
36
+ import { dirname } from '@visulima/path';
37
+
38
+ const REMOTE_EXECUTION_PROTO = `syntax = "proto3";
39
+
40
+ package build.bazel.remote.execution.v2;
41
+
42
+ message Digest {
43
+ string hash = 1;
44
+ int64 size_bytes = 2;
45
+ }
46
+
47
+ message NodeProperties {
48
+ repeated NodeProperty properties = 1;
49
+ string mtime = 2;
50
+ uint32 unix_mode = 3;
51
+ }
52
+
53
+ message NodeProperty {
54
+ string name = 1;
55
+ string value = 2;
56
+ }
57
+
58
+ message OutputFile {
59
+ string path = 1;
60
+ Digest digest = 2;
61
+ bool is_executable = 4;
62
+ bytes contents = 5;
63
+ NodeProperties node_properties = 7;
64
+ }
65
+
66
+ message OutputDirectory {
67
+ string path = 1;
68
+ Digest tree_digest = 3;
69
+ bool is_topologically_sorted = 4;
70
+ Digest root_directory_digest = 5;
71
+ }
72
+
73
+ message OutputSymlink {
74
+ string path = 1;
75
+ string target = 2;
76
+ NodeProperties node_properties = 3;
77
+ }
78
+
79
+ message ActionResult {
80
+ repeated OutputFile output_files = 2;
81
+ repeated OutputSymlink output_file_symlinks = 10;
82
+ repeated OutputSymlink output_symlinks = 12;
83
+ repeated OutputDirectory output_directories = 3;
84
+ repeated OutputSymlink output_directory_symlinks = 11;
85
+ int32 exit_code = 4;
86
+ bytes stdout_raw = 5;
87
+ Digest stdout_digest = 6;
88
+ bytes stderr_raw = 7;
89
+ Digest stderr_digest = 8;
90
+ }
91
+
92
+ message FindMissingBlobsRequest {
93
+ string instance_name = 1;
94
+ repeated Digest blob_digests = 2;
95
+ DigestFunction.Value digest_function = 3;
96
+ }
97
+
98
+ message FindMissingBlobsResponse {
99
+ repeated Digest missing_blob_digests = 2;
100
+ }
101
+
102
+ message BatchUpdateBlobsRequest {
103
+ message Request {
104
+ Digest digest = 1;
105
+ bytes data = 2;
106
+ Compressor.Value compressor = 3;
107
+ }
108
+ string instance_name = 1;
109
+ repeated Request requests = 2;
110
+ DigestFunction.Value digest_function = 5;
111
+ }
112
+
113
+ message BatchUpdateBlobsResponse {
114
+ message Response {
115
+ Digest digest = 1;
116
+ Status status = 2;
117
+ }
118
+ repeated Response responses = 1;
119
+ }
120
+
121
+ message BatchReadBlobsRequest {
122
+ string instance_name = 1;
123
+ repeated Digest digests = 2;
124
+ repeated Compressor.Value acceptable_compressors = 3;
125
+ DigestFunction.Value digest_function = 4;
126
+ }
127
+
128
+ message BatchReadBlobsResponse {
129
+ message Response {
130
+ Digest digest = 1;
131
+ bytes data = 2;
132
+ Compressor.Value compressor = 4;
133
+ Status status = 3;
134
+ }
135
+ repeated Response responses = 1;
136
+ }
137
+
138
+ message GetActionResultRequest {
139
+ string instance_name = 1;
140
+ Digest action_digest = 2;
141
+ bool inline_stdout = 3;
142
+ bool inline_stderr = 4;
143
+ repeated string inline_output_files = 5;
144
+ DigestFunction.Value digest_function = 6;
145
+ }
146
+
147
+ message UpdateActionResultRequest {
148
+ string instance_name = 1;
149
+ Digest action_digest = 2;
150
+ ActionResult action_result = 3;
151
+ ResultsCachePolicy results_cache_policy = 4;
152
+ DigestFunction.Value digest_function = 5;
153
+ }
154
+
155
+ message ResultsCachePolicy {
156
+ int32 priority = 1;
157
+ }
158
+
159
+ message GetCapabilitiesRequest {
160
+ string instance_name = 1;
161
+ }
162
+
163
+ message ServerCapabilities {
164
+ CacheCapabilities cache_capabilities = 1;
165
+ ExecutionCapabilities execution_capabilities = 2;
166
+ SemVer deprecated_api_version = 3;
167
+ SemVer low_api_version = 4;
168
+ SemVer high_api_version = 5;
169
+ }
170
+
171
+ message ExecutionCapabilities {
172
+ DigestFunction.Value digest_function = 1;
173
+ bool exec_enabled = 2;
174
+ PriorityCapabilities execution_priority_capabilities = 3;
175
+ repeated string supported_node_properties = 4;
176
+ repeated DigestFunction.Value digest_functions = 5;
177
+ }
178
+
179
+ message CacheCapabilities {
180
+ repeated DigestFunction.Value digest_functions = 1;
181
+ ActionCacheUpdateCapabilities action_cache_update_capabilities = 2;
182
+ PriorityCapabilities cache_priority_capabilities = 3;
183
+ int64 max_batch_total_size_bytes = 4;
184
+ SymlinkAbsolutePathStrategy.Value symlink_absolute_path_strategy = 5;
185
+ repeated Compressor.Value supported_compressors = 6;
186
+ repeated Compressor.Value supported_batch_update_compressors = 7;
187
+ }
188
+
189
+ message ActionCacheUpdateCapabilities {
190
+ bool update_enabled = 1;
191
+ }
192
+
193
+ message PriorityCapabilities {
194
+ message PriorityRange {
195
+ int32 min_priority = 1;
196
+ int32 max_priority = 2;
197
+ }
198
+ repeated PriorityRange priorities = 1;
199
+ }
200
+
201
+ message SymlinkAbsolutePathStrategy {
202
+ enum Value {
203
+ UNKNOWN = 0;
204
+ DISALLOWED = 1;
205
+ ALLOWED = 2;
206
+ }
207
+ }
208
+
209
+ message DigestFunction {
210
+ enum Value {
211
+ UNKNOWN = 0;
212
+ SHA256 = 1;
213
+ SHA1 = 2;
214
+ MD5 = 3;
215
+ VSO = 4;
216
+ SHA384 = 5;
217
+ SHA512 = 6;
218
+ MURMUR3 = 7;
219
+ SHA256TREE = 8;
220
+ BLAKE3 = 9;
221
+ }
222
+ }
223
+
224
+ message Compressor {
225
+ enum Value {
226
+ IDENTITY = 0;
227
+ ZSTD = 1;
228
+ DEFLATE = 2;
229
+ BROTLI = 3;
230
+ }
231
+ }
232
+
233
+ message SemVer {
234
+ int32 major = 1;
235
+ int32 minor = 2;
236
+ string patch = 3;
237
+ string prerelease = 4;
238
+ }
239
+
240
+ message Status {
241
+ int32 code = 1;
242
+ string message = 2;
243
+ }
244
+
245
+ service ContentAddressableStorage {
246
+ rpc FindMissingBlobs(FindMissingBlobsRequest) returns (FindMissingBlobsResponse);
247
+ rpc BatchUpdateBlobs(BatchUpdateBlobsRequest) returns (BatchUpdateBlobsResponse);
248
+ rpc BatchReadBlobs(BatchReadBlobsRequest) returns (BatchReadBlobsResponse);
249
+ }
250
+
251
+ service ActionCache {
252
+ rpc GetActionResult(GetActionResultRequest) returns (ActionResult);
253
+ rpc UpdateActionResult(UpdateActionResultRequest) returns (ActionResult);
254
+ }
255
+
256
+ service Capabilities {
257
+ rpc GetCapabilities(GetCapabilitiesRequest) returns (ServerCapabilities);
258
+ }
259
+ `;
260
+ const BYTESTREAM_PROTO = `syntax = "proto3";
261
+
262
+ package google.bytestream;
263
+
264
+ message ReadRequest {
265
+ string resource_name = 1;
266
+ int64 read_offset = 2;
267
+ int64 read_limit = 3;
268
+ }
269
+
270
+ message ReadResponse {
271
+ bytes data = 10;
272
+ }
273
+
274
+ message WriteRequest {
275
+ string resource_name = 1;
276
+ int64 write_offset = 2;
277
+ bool finish_write = 3;
278
+ bytes data = 10;
279
+ }
280
+
281
+ message WriteResponse {
282
+ int64 committed_size = 1;
283
+ }
284
+
285
+ service ByteStream {
286
+ rpc Read(ReadRequest) returns (stream ReadResponse);
287
+ rpc Write(stream WriteRequest) returns (WriteResponse);
288
+ }
289
+ `;
290
+
291
+ let cached;
292
+ const importGrpc = async () => {
293
+ let grpc;
294
+ let protoLoader;
295
+ try {
296
+ grpc = await import('@grpc/grpc-js');
297
+ } catch (error) {
298
+ throw new Error(
299
+ '[task-runner] remoteCache.backend = "reapi" needs the optional peer dependency `@grpc/grpc-js`. Install it with `pnpm add @grpc/grpc-js` (or your package manager\'s equivalent), or switch to backend: "http".',
300
+ { cause: error }
301
+ );
302
+ }
303
+ try {
304
+ protoLoader = await import('@grpc/proto-loader');
305
+ } catch (error) {
306
+ throw new Error(
307
+ '[task-runner] remoteCache.backend = "reapi" needs the optional peer dependency `@grpc/proto-loader`. Install it with `pnpm add @grpc/proto-loader` (or your package manager\'s equivalent), or switch to backend: "http".',
308
+ { cause: error }
309
+ );
310
+ }
311
+ return { grpc, protoLoader };
312
+ };
313
+ const loadReapiProto = async () => {
314
+ const { grpc, protoLoader } = await importGrpc();
315
+ if (cached === void 0) {
316
+ let protobufjs;
317
+ try {
318
+ protobufjs = await import('../packem_chunks/index.js').then(n => n.i);
319
+ } catch (error) {
320
+ throw new Error(
321
+ '[task-runner] remoteCache.backend = "reapi" needs `protobufjs` (normally installed transitively via `@grpc/proto-loader`). Install it with `pnpm add protobufjs` (or your package manager\'s equivalent) if your installer does not hoist transitive deps.',
322
+ { cause: error }
323
+ );
324
+ }
325
+ const { parse, Root } = protobufjs;
326
+ const root = new Root();
327
+ parse(REMOTE_EXECUTION_PROTO, root, { keepCase: true });
328
+ parse(BYTESTREAM_PROTO, root, { keepCase: true });
329
+ root.resolveAll();
330
+ const json = root.toJSON();
331
+ const definition = protoLoader.fromJSON(json, {
332
+ arrays: true,
333
+ defaults: true,
334
+ enums: String,
335
+ keepCase: true,
336
+ longs: Number,
337
+ objects: true,
338
+ oneofs: true
339
+ });
340
+ const loaded = grpc.loadPackageDefinition(definition);
341
+ const reapi = loaded.build.bazel.remote.execution.v2;
342
+ const byteStream = loaded.google.bytestream;
343
+ cached = {
344
+ clients: {
345
+ ActionCache: reapi.ActionCache,
346
+ ByteStream: byteStream.ByteStream,
347
+ Capabilities: reapi.Capabilities,
348
+ ContentAddressableStorage: reapi.ContentAddressableStorage
349
+ },
350
+ definition
351
+ };
352
+ }
353
+ return { clients: cached.clients, grpc };
354
+ };
355
+
356
+ const COMPRESSOR_IDENTITY = "IDENTITY";
357
+ const STATUS_OK = 0;
358
+ const STATUS_NOT_FOUND = 5;
359
+ const STATUS_UNAUTHENTICATED = 16;
360
+ const STATUS_PERMISSION_DENIED = 7;
361
+ const DEFAULT_MAX_BATCH_TOTAL_SIZE_BYTES = 4 * 1024 * 1024;
362
+ const STREAM_CHUNK_SIZE = 64 * 1024;
363
+ const BATCH_ENTRY_OVERHEAD_BYTES = 256;
364
+ const STREAM_DEADLINE_MULTIPLIER = 10;
365
+ const buildReadResourceName = (instanceName, digest) => {
366
+ const prefix = instanceName === "" ? "" : `${instanceName}/`;
367
+ return `${prefix}blobs/${digest.hash}/${String(digest.sizeBytes)}`;
368
+ };
369
+ const buildWriteResourceName = (instanceName, digest) => {
370
+ const prefix = instanceName === "" ? "" : `${instanceName}/`;
371
+ return `${prefix}uploads/${randomUUID()}/blobs/${digest.hash}/${String(digest.sizeBytes)}`;
372
+ };
373
+ const callUnary = async (client, method, request, metadata, deadlineMs) => {
374
+ const fn = client[method];
375
+ if (typeof fn !== "function") {
376
+ throw new TypeError(`[task-runner] REAPI client is missing method ${method} — proto descriptor mismatch.`);
377
+ }
378
+ return new Promise((resolve, reject) => {
379
+ const callOptions = { deadline: Date.now() + deadlineMs };
380
+ fn.call(
381
+ client,
382
+ request,
383
+ metadata,
384
+ callOptions,
385
+ (error, response) => {
386
+ if (error) {
387
+ reject(error);
388
+ return;
389
+ }
390
+ resolve(response);
391
+ }
392
+ );
393
+ });
394
+ };
395
+ class ReapiRemoteCache {
396
+ #target;
397
+ #useTls;
398
+ #bearerToken;
399
+ #instanceName;
400
+ #read;
401
+ #write;
402
+ #timeout;
403
+ #onUploadError;
404
+ #inflightUploads = /* @__PURE__ */ new Map();
405
+ #clientsPromise;
406
+ #capabilities;
407
+ constructor(options) {
408
+ const { target, useTls } = parseGrpcUrl(options.url);
409
+ this.#target = target;
410
+ this.#useTls = useTls;
411
+ this.#bearerToken = options.bearerToken;
412
+ this.#instanceName = options.instanceName ?? "";
413
+ this.#timeout = options.timeout ?? 3e4;
414
+ this.#onUploadError = options.onUploadError;
415
+ const mode = options.mode ?? "readwrite";
416
+ this.#read = mode === "read" || mode === "readwrite";
417
+ this.#write = mode === "write" || mode === "readwrite";
418
+ if (this.#bearerToken !== void 0 && !this.#useTls && options.allowInsecureBearer !== true) {
419
+ throw new Error(
420
+ '[task-runner] remoteCache.backend = "reapi" refuses to send a bearer token over cleartext gRPC. Use `grpcs://` (or terminate TLS at a reverse proxy), or pass `allowInsecureBearer: true` for trusted-boundary deployments (loopback, mesh mTLS sidecar).'
421
+ );
422
+ }
423
+ }
424
+ /**
425
+ * Close all gRPC channels held by this backend. Idempotent — safe
426
+ * to call multiple times, and safe to call before any RPC was
427
+ * issued. If `#getClients` is currently in flight we await its
428
+ * resolution so the underlying channels are observable to close.
429
+ */
430
+ async close() {
431
+ const pending = this.#clientsPromise;
432
+ if (pending === void 0) {
433
+ return;
434
+ }
435
+ this.#clientsPromise = void 0;
436
+ let resolved;
437
+ try {
438
+ resolved = await pending;
439
+ } catch {
440
+ return;
441
+ }
442
+ for (const client of [resolved.actionCache, resolved.byteStream, resolved.capabilities, resolved.cas]) {
443
+ try {
444
+ client.close();
445
+ } catch {
446
+ }
447
+ }
448
+ }
449
+ /**
450
+ * Diagnostic probe — fetches the server's `Capabilities` RPC response
451
+ * (or the cached value, if a previous call already negotiated). Used
452
+ * by `vis cache doctor` to surface what the server advertises without
453
+ * forcing the operator to issue a real CAS RPC.
454
+ *
455
+ * Bypasses the read/write mode gate intentionally: a probe must work
456
+ * even on a cache configured `mode: "write"` so the operator can
457
+ * verify the connection regardless of how the runner uses it.
458
+ */
459
+ async probeCapabilities() {
460
+ return this.#negotiateCapabilities();
461
+ }
462
+ async containsAction(actionDigest) {
463
+ if (!this.#read) {
464
+ return false;
465
+ }
466
+ await this.#assertSha256Supported();
467
+ try {
468
+ const { actionCache } = await this.#getClients();
469
+ const metadata = await this.#buildMetadata();
470
+ await callUnary(
471
+ actionCache,
472
+ "GetActionResult",
473
+ {
474
+ action_digest: digestToProto(actionDigest),
475
+ inline_stderr: false,
476
+ inline_stdout: false,
477
+ instance_name: this.#instanceName
478
+ },
479
+ metadata,
480
+ this.#timeout
481
+ );
482
+ return true;
483
+ } catch (error) {
484
+ if (isNotFoundError(error)) {
485
+ return false;
486
+ }
487
+ throw error;
488
+ }
489
+ }
490
+ async fetchBlob(digest, destinationPath) {
491
+ if (!this.#read) {
492
+ return false;
493
+ }
494
+ await this.#assertSha256Supported();
495
+ try {
496
+ const maxBatchSize = await this.#getMaxBatchSize();
497
+ await mkdir(dirname(destinationPath), { recursive: true });
498
+ if (digest.sizeBytes <= maxBatchSize) {
499
+ const bytes = await this.#batchReadOne(digest);
500
+ if (bytes === void 0) {
501
+ return false;
502
+ }
503
+ await pipeline(Readable.from(bytes), createWriteStream(destinationPath));
504
+ return true;
505
+ }
506
+ return await this.#streamRead(digest, destinationPath);
507
+ } catch {
508
+ await rm(destinationPath, { force: true }).catch(() => {
509
+ });
510
+ return false;
511
+ }
512
+ }
513
+ async retrieveAction(actionDigest) {
514
+ if (!this.#read) {
515
+ return null;
516
+ }
517
+ await this.#assertSha256Supported();
518
+ try {
519
+ const { actionCache } = await this.#getClients();
520
+ const metadata = await this.#buildMetadata();
521
+ const proto = await callUnary(
522
+ actionCache,
523
+ "GetActionResult",
524
+ {
525
+ action_digest: digestToProto(actionDigest),
526
+ inline_stderr: false,
527
+ inline_stdout: false,
528
+ instance_name: this.#instanceName
529
+ },
530
+ metadata,
531
+ this.#timeout
532
+ );
533
+ return protoToActionResult(proto);
534
+ } catch (error) {
535
+ if (isNotFoundError(error)) {
536
+ return null;
537
+ }
538
+ throw error;
539
+ }
540
+ }
541
+ async storeAction(actionDigest, result, blobs) {
542
+ if (!this.#write) {
543
+ return false;
544
+ }
545
+ await this.#assertSha256Supported();
546
+ try {
547
+ const missing = await this.#findMissingBlobs(blobs.map((blob) => blob.digest));
548
+ const missingSet = new Set(missing.map((digest) => digest.hash));
549
+ const blobsToUpload = blobs.filter((blob) => missingSet.has(blob.digest.hash));
550
+ await this.#uploadBlobs(blobsToUpload);
551
+ const { actionCache } = await this.#getClients();
552
+ const metadata = await this.#buildMetadata();
553
+ await callUnary(
554
+ actionCache,
555
+ "UpdateActionResult",
556
+ {
557
+ action_digest: digestToProto(actionDigest),
558
+ action_result: actionResultToProto(result),
559
+ instance_name: this.#instanceName
560
+ },
561
+ metadata,
562
+ this.#timeout
563
+ );
564
+ return true;
565
+ } catch (error) {
566
+ this.#onUploadError?.(actionDigest.hash, error);
567
+ return false;
568
+ }
569
+ }
570
+ async #getClients() {
571
+ if (this.#clientsPromise === void 0) {
572
+ this.#clientsPromise = (async () => {
573
+ const { clients, grpc } = await loadReapiProto();
574
+ const credentials = this.#useTls ? grpc.credentials.createSsl() : grpc.credentials.createInsecure();
575
+ return {
576
+ actionCache: new clients.ActionCache(this.#target, credentials),
577
+ byteStream: new clients.ByteStream(this.#target, credentials),
578
+ capabilities: new clients.Capabilities(this.#target, credentials),
579
+ cas: new clients.ContentAddressableStorage(this.#target, credentials),
580
+ clients,
581
+ grpc
582
+ };
583
+ })();
584
+ }
585
+ return this.#clientsPromise;
586
+ }
587
+ async #buildMetadata() {
588
+ const { grpc } = await this.#getClients();
589
+ const metadata = new grpc.Metadata();
590
+ if (this.#bearerToken !== void 0) {
591
+ metadata.set("authorization", `Bearer ${this.#bearerToken}`);
592
+ }
593
+ return metadata;
594
+ }
595
+ async #getMaxBatchSize() {
596
+ const negotiated = await this.#negotiateCapabilities();
597
+ return negotiated.maxBatchTotalSizeBytes;
598
+ }
599
+ /**
600
+ * One-shot capability negotiation. Caches the result so the
601
+ * first {@link RemoteCacheBackend} method to need it pays the
602
+ * round-trip and every subsequent call sees the cached values.
603
+ *
604
+ * Auth failures (`UNAUTHENTICATED` / `PERMISSION_DENIED`) are
605
+ * re-thrown — falling back silently would let a misconfigured
606
+ * token survive until the next CAS RPC, where the failure mode is
607
+ * less obviously "fix your auth". `NOT_FOUND` and benign network
608
+ * errors degrade to defaults so the backend stays usable against
609
+ * older REAPI servers that do not implement Capabilities.
610
+ *
611
+ * `digest_functions` is recorded so `#assertSha256Supported`
612
+ * can refuse to talk to a server that only advertises non-sha256
613
+ * digests — vis pins sha256 for action digests and a server
614
+ * expecting BLAKE3 would reject every request with
615
+ * `INVALID_ARGUMENT`.
616
+ */
617
+ async #negotiateCapabilities() {
618
+ if (this.#capabilities !== void 0) {
619
+ return this.#capabilities;
620
+ }
621
+ try {
622
+ const { capabilities } = await this.#getClients();
623
+ const metadata = await this.#buildMetadata();
624
+ const response = await callUnary(
625
+ capabilities,
626
+ "GetCapabilities",
627
+ { instance_name: this.#instanceName },
628
+ metadata,
629
+ this.#timeout
630
+ );
631
+ const cacheCaps = response?.cache_capabilities;
632
+ const advertised = cacheCaps && typeof cacheCaps.max_batch_total_size_bytes === "number" ? cacheCaps.max_batch_total_size_bytes : 0;
633
+ const digestFunctions = (cacheCaps?.digest_functions ?? []).map(String);
634
+ this.#capabilities = {
635
+ digestFunctions,
636
+ maxBatchTotalSizeBytes: advertised > 0 ? advertised : DEFAULT_MAX_BATCH_TOTAL_SIZE_BYTES
637
+ };
638
+ return this.#capabilities;
639
+ } catch (error) {
640
+ if (isAuthError(error)) {
641
+ throw error;
642
+ }
643
+ this.#capabilities = { digestFunctions: [], maxBatchTotalSizeBytes: DEFAULT_MAX_BATCH_TOTAL_SIZE_BYTES };
644
+ return this.#capabilities;
645
+ }
646
+ }
647
+ /**
648
+ * Refuse to issue CAS / ActionCache RPCs against a server whose
649
+ * advertised digest functions don't include sha256. Skipped when
650
+ * negotiation found an empty list (older / non-conforming
651
+ * servers) — there we fall back to "try it and let the server
652
+ * reject" rather than refusing outright.
653
+ */
654
+ async #assertSha256Supported() {
655
+ const { digestFunctions } = await this.#negotiateCapabilities();
656
+ if (digestFunctions.length === 0) {
657
+ return;
658
+ }
659
+ const supportsSha256 = digestFunctions.some((name) => name.toUpperCase() === "SHA256");
660
+ if (!supportsSha256) {
661
+ throw new Error(
662
+ `[task-runner] REAPI server does not advertise SHA256 in cache_capabilities.digest_functions (got: ${digestFunctions.join(", ")}). vis pins sha256 for action digests; a server expecting a different digest function would reject every request.`
663
+ );
664
+ }
665
+ }
666
+ async #findMissingBlobs(digests) {
667
+ if (digests.length === 0) {
668
+ return [];
669
+ }
670
+ const { cas } = await this.#getClients();
671
+ const metadata = await this.#buildMetadata();
672
+ const response = await callUnary(
673
+ cas,
674
+ "FindMissingBlobs",
675
+ {
676
+ blob_digests: digests.map((digest) => digestToProto(digest)),
677
+ instance_name: this.#instanceName
678
+ },
679
+ metadata,
680
+ this.#timeout
681
+ );
682
+ const missing = response?.missing_blob_digests ?? [];
683
+ return missing.map((proto) => protoToDigest(proto));
684
+ }
685
+ async #uploadBlobs(blobs) {
686
+ if (blobs.length === 0) {
687
+ return;
688
+ }
689
+ const maxBatchSize = await this.#getMaxBatchSize();
690
+ const perBlobLimit = maxBatchSize - BATCH_ENTRY_OVERHEAD_BYTES;
691
+ const small = [];
692
+ const large = [];
693
+ for (const blob of blobs) {
694
+ (blob.digest.sizeBytes <= perBlobLimit ? small : large).push(blob);
695
+ }
696
+ const batches = bucketIntoBatches(small, maxBatchSize);
697
+ for (const batch of batches) {
698
+ await this.#uploadBatchOnce(batch);
699
+ }
700
+ for (const blob of large) {
701
+ await this.#uploadStreamOnce(blob);
702
+ }
703
+ }
704
+ async #uploadBatchOnce(blobs) {
705
+ const dedupedBlobs = [];
706
+ const dedupedPromises = [];
707
+ for (const blob of blobs) {
708
+ const inflight = this.#inflightUploads.get(blob.digest.hash);
709
+ if (inflight !== void 0) {
710
+ dedupedPromises.push(inflight);
711
+ continue;
712
+ }
713
+ dedupedBlobs.push(blob);
714
+ }
715
+ if (dedupedBlobs.length > 0) {
716
+ const promise = this.#uploadBatchToWire(dedupedBlobs);
717
+ for (const blob of dedupedBlobs) {
718
+ this.#inflightUploads.set(blob.digest.hash, promise);
719
+ }
720
+ try {
721
+ await promise;
722
+ } finally {
723
+ for (const blob of dedupedBlobs) {
724
+ this.#inflightUploads.delete(blob.digest.hash);
725
+ }
726
+ }
727
+ }
728
+ await Promise.all(dedupedPromises);
729
+ }
730
+ async #uploadBatchToWire(blobs) {
731
+ const requests = [];
732
+ for (const blob of blobs) {
733
+ const data = await streamToBuffer(await blob.open());
734
+ requests.push({
735
+ compressor: COMPRESSOR_IDENTITY,
736
+ data,
737
+ digest: digestToProto(blob.digest)
738
+ });
739
+ }
740
+ const { cas } = await this.#getClients();
741
+ const metadata = await this.#buildMetadata();
742
+ const response = await callUnary(
743
+ cas,
744
+ "BatchUpdateBlobs",
745
+ {
746
+ instance_name: this.#instanceName,
747
+ requests
748
+ },
749
+ metadata,
750
+ this.#timeout
751
+ );
752
+ const responses = response?.responses ?? [];
753
+ for (const entry of responses) {
754
+ const code = entry.status.code ?? 0;
755
+ if (code !== STATUS_OK) {
756
+ throw new Error(`[task-runner] BatchUpdateBlobs reported code ${String(code)} for ${entry.digest.hash}: ${entry.status.message ?? ""}`);
757
+ }
758
+ }
759
+ return true;
760
+ }
761
+ async #uploadStreamOnce(blob) {
762
+ const inflight = this.#inflightUploads.get(blob.digest.hash);
763
+ if (inflight !== void 0) {
764
+ await inflight;
765
+ return;
766
+ }
767
+ const promise = this.#streamWrite(blob);
768
+ this.#inflightUploads.set(blob.digest.hash, promise);
769
+ try {
770
+ await promise;
771
+ } finally {
772
+ this.#inflightUploads.delete(blob.digest.hash);
773
+ }
774
+ }
775
+ async #streamWrite(blob) {
776
+ const { byteStream } = await this.#getClients();
777
+ const metadata = await this.#buildMetadata();
778
+ const resourceName = buildWriteResourceName(this.#instanceName, blob.digest);
779
+ const writeFn = byteStream.Write;
780
+ if (typeof writeFn !== "function") {
781
+ throw new TypeError("[task-runner] REAPI ByteStream client is missing Write method.");
782
+ }
783
+ await new Promise((resolve, reject) => {
784
+ const callOptions = { deadline: Date.now() + this.#timeout * STREAM_DEADLINE_MULTIPLIER };
785
+ const call = writeFn.call(byteStream, metadata, callOptions, (error) => {
786
+ if (error) {
787
+ reject(error);
788
+ return;
789
+ }
790
+ resolve();
791
+ });
792
+ void (async () => {
793
+ try {
794
+ let writeOffset = 0;
795
+ let firstChunk = true;
796
+ const stream = await blob.open();
797
+ for await (const rawChunk of stream) {
798
+ const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk);
799
+ for (let pos = 0; pos < chunk.byteLength; pos += STREAM_CHUNK_SIZE) {
800
+ const slice = chunk.subarray(pos, pos + STREAM_CHUNK_SIZE);
801
+ call.write({
802
+ data: slice,
803
+ finish_write: false,
804
+ resource_name: firstChunk ? resourceName : "",
805
+ write_offset: writeOffset
806
+ });
807
+ writeOffset += slice.byteLength;
808
+ firstChunk = false;
809
+ }
810
+ }
811
+ call.write({
812
+ data: Buffer.alloc(0),
813
+ finish_write: true,
814
+ resource_name: firstChunk ? resourceName : "",
815
+ write_offset: writeOffset
816
+ });
817
+ call.end();
818
+ } catch (error) {
819
+ try {
820
+ call.cancel?.();
821
+ } catch {
822
+ }
823
+ reject(error);
824
+ }
825
+ })();
826
+ });
827
+ return true;
828
+ }
829
+ async #batchReadOne(digest) {
830
+ const { cas } = await this.#getClients();
831
+ const metadata = await this.#buildMetadata();
832
+ const response = await callUnary(
833
+ cas,
834
+ "BatchReadBlobs",
835
+ {
836
+ acceptable_compressors: [COMPRESSOR_IDENTITY],
837
+ digests: [digestToProto(digest)],
838
+ instance_name: this.#instanceName
839
+ },
840
+ metadata,
841
+ this.#timeout
842
+ );
843
+ const entry = response?.responses?.[0];
844
+ if (!entry) {
845
+ return void 0;
846
+ }
847
+ const code = entry.status.code ?? 0;
848
+ if (code !== STATUS_OK) {
849
+ return void 0;
850
+ }
851
+ if (entry.data === void 0) {
852
+ return Buffer.alloc(0);
853
+ }
854
+ return Buffer.isBuffer(entry.data) ? entry.data : Buffer.from(entry.data);
855
+ }
856
+ async #streamRead(digest, destinationPath) {
857
+ const { byteStream } = await this.#getClients();
858
+ const metadata = await this.#buildMetadata();
859
+ const resourceName = buildReadResourceName(this.#instanceName, digest);
860
+ const readFn = byteStream.Read;
861
+ if (typeof readFn !== "function") {
862
+ throw new TypeError("[task-runner] REAPI ByteStream client is missing Read method.");
863
+ }
864
+ const callOptions = { deadline: Date.now() + this.#timeout * STREAM_DEADLINE_MULTIPLIER };
865
+ const call = readFn.call(
866
+ byteStream,
867
+ { read_limit: 0, read_offset: 0, resource_name: resourceName },
868
+ metadata,
869
+ callOptions
870
+ );
871
+ const writer = createWriteStream(destinationPath);
872
+ try {
873
+ for await (const message of call) {
874
+ const { data } = message;
875
+ if (data === void 0) {
876
+ continue;
877
+ }
878
+ if (!writer.write(Buffer.isBuffer(data) ? data : Buffer.from(data))) {
879
+ await new Promise((resolve) => {
880
+ writer.once("drain", () => {
881
+ resolve();
882
+ });
883
+ });
884
+ }
885
+ }
886
+ await new Promise((resolve, reject) => {
887
+ writer.end((error) => {
888
+ if (error) {
889
+ reject(error);
890
+ return;
891
+ }
892
+ resolve();
893
+ });
894
+ });
895
+ return true;
896
+ } catch (error) {
897
+ writer.destroy();
898
+ if (isNotFoundError(error)) {
899
+ return false;
900
+ }
901
+ throw error;
902
+ }
903
+ }
904
+ }
905
+ const digestToProto = (digest) => {
906
+ return {
907
+ hash: digest.hash,
908
+ size_bytes: digest.sizeBytes
909
+ };
910
+ };
911
+ const protoToDigest = (proto) => {
912
+ return {
913
+ hash: proto?.hash ?? "",
914
+ sizeBytes: typeof proto?.size_bytes === "number" ? proto.size_bytes : 0
915
+ };
916
+ };
917
+ const protoToActionResult = (proto) => {
918
+ return {
919
+ exitCode: proto?.exit_code ?? 0,
920
+ outputDirectories: (proto?.output_directories ?? []).map((entry) => {
921
+ return {
922
+ path: entry.path,
923
+ treeDigest: protoToDigest(entry.tree_digest)
924
+ };
925
+ }),
926
+ outputFiles: (proto?.output_files ?? []).map((entry) => {
927
+ return {
928
+ digest: protoToDigest(entry.digest),
929
+ isExecutable: entry.is_executable ?? false,
930
+ path: entry.path
931
+ };
932
+ }),
933
+ stdoutDigest: proto?.stdout_digest === void 0 ? void 0 : protoToDigest(proto.stdout_digest)
934
+ };
935
+ };
936
+ const actionResultToProto = (result) => {
937
+ return {
938
+ exit_code: result.exitCode,
939
+ output_directories: result.outputDirectories.map((entry) => {
940
+ return {
941
+ path: entry.path,
942
+ tree_digest: digestToProto(entry.treeDigest)
943
+ };
944
+ }),
945
+ output_files: result.outputFiles.map((entry) => {
946
+ return {
947
+ digest: digestToProto(entry.digest),
948
+ is_executable: entry.isExecutable,
949
+ path: entry.path
950
+ };
951
+ }),
952
+ stdout_digest: result.stdoutDigest === void 0 ? void 0 : digestToProto(result.stdoutDigest)
953
+ };
954
+ };
955
+ const isNotFoundError = (error) => {
956
+ if (typeof error !== "object" || error === null) {
957
+ return false;
958
+ }
959
+ const { code } = error;
960
+ return code === STATUS_NOT_FOUND;
961
+ };
962
+ const isAuthError = (error) => {
963
+ if (typeof error !== "object" || error === null) {
964
+ return false;
965
+ }
966
+ const { code } = error;
967
+ return code === STATUS_UNAUTHENTICATED || code === STATUS_PERMISSION_DENIED;
968
+ };
969
+ const parseGrpcUrl = (url) => {
970
+ const trimmed = url.trim();
971
+ if (trimmed === "") {
972
+ throw new Error("[task-runner] REAPI backend requires a non-empty `url`.");
973
+ }
974
+ const match = /^(grpcs?):\/\/(.+)$/i.exec(trimmed);
975
+ if (match === null) {
976
+ return { target: trimmed, useTls: false };
977
+ }
978
+ const [, scheme, target] = match;
979
+ const finalTarget = (target ?? "").trim();
980
+ if (finalTarget === "") {
981
+ throw new Error(`[task-runner] REAPI backend url ${url} is missing a host:port target.`);
982
+ }
983
+ return { target: finalTarget, useTls: scheme?.toLowerCase() === "grpcs" };
984
+ };
985
+ const bucketIntoBatches = (blobs, maxBatchSize) => {
986
+ const batches = [];
987
+ let current = [];
988
+ let currentSize = 0;
989
+ for (const blob of blobs) {
990
+ const entrySize = blob.digest.sizeBytes + BATCH_ENTRY_OVERHEAD_BYTES;
991
+ if (currentSize + entrySize > maxBatchSize && current.length > 0) {
992
+ batches.push(current);
993
+ current = [];
994
+ currentSize = 0;
995
+ }
996
+ current.push(blob);
997
+ currentSize += entrySize;
998
+ }
999
+ if (current.length > 0) {
1000
+ batches.push(current);
1001
+ }
1002
+ return batches;
1003
+ };
1004
+ const streamToBuffer = async (source) => {
1005
+ const chunks = [];
1006
+ for await (const chunk of source) {
1007
+ chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
1008
+ }
1009
+ return Buffer.concat(chunks);
1010
+ };
1011
+
1012
+ export { ReapiRemoteCache };