@smoothbricks/nx-plugin 0.7.18 → 0.7.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/README.md +160 -99
  2. package/dist/.build-inputs.json +35 -29
  3. package/dist/built-output-packages.js +1 -1
  4. package/dist/cargo-closure-input.js +1 -1
  5. package/dist/cargo-source-hash.d.ts +3 -21
  6. package/dist/cargo-source-hash.d.ts.map +1 -1
  7. package/dist/cargo-source-hash.js +4 -91
  8. package/dist/cargo-toolchain-paths.d.ts +2 -2
  9. package/dist/cargo-toolchain-paths.js +2 -2
  10. package/dist/cargo-workspace.d.ts.map +1 -1
  11. package/dist/cargo-workspace.js +2 -1
  12. package/dist/executors/bounded-exec/admission-queue.d.ts +3 -1
  13. package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -1
  14. package/dist/executors/bounded-exec/admission-queue.js +9 -10
  15. package/dist/executors/bounded-exec/awake-clock.d.ts +5 -0
  16. package/dist/executors/bounded-exec/awake-clock.d.ts.map +1 -0
  17. package/dist/executors/bounded-exec/awake-clock.js +22 -0
  18. package/dist/executors/bounded-exec/bounds.d.ts +7 -5
  19. package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
  20. package/dist/executors/bounded-exec/bounds.js +14 -11
  21. package/dist/executors/bounded-exec/case-gate.d.ts +114 -0
  22. package/dist/executors/bounded-exec/case-gate.d.ts.map +1 -0
  23. package/dist/executors/bounded-exec/case-gate.js +342 -0
  24. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +1 -0
  25. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
  26. package/dist/executors/bounded-exec/cpu-pool-thread.js +2 -2
  27. package/dist/executors/bounded-exec/cpu-pool.d.ts +28 -1
  28. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  29. package/dist/executors/bounded-exec/cpu-pool.js +102 -16
  30. package/dist/executors/bounded-exec/endpoint-life.d.ts +146 -0
  31. package/dist/executors/bounded-exec/endpoint-life.d.ts.map +1 -0
  32. package/dist/executors/bounded-exec/endpoint-life.js +228 -0
  33. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  34. package/dist/executors/bounded-exec/executor.js +207 -69
  35. package/dist/executors/bounded-exec/gateway-lease.d.ts +1 -1
  36. package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
  37. package/dist/executors/bounded-exec/gateway-lease.js +8 -1
  38. package/dist/executors/bounded-exec/go-run-summary.d.ts +51 -0
  39. package/dist/executors/bounded-exec/go-run-summary.d.ts.map +1 -0
  40. package/dist/executors/bounded-exec/go-run-summary.js +98 -0
  41. package/dist/executors/bounded-exec/hang-diagnostics.d.ts +3 -3
  42. package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
  43. package/dist/executors/bounded-exec/hang-diagnostics.js +66 -48
  44. package/dist/executors/bounded-exec/lease-wait.d.ts +1 -0
  45. package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -1
  46. package/dist/executors/bounded-exec/lease-wait.js +1 -0
  47. package/dist/executors/bounded-exec/mbx-pool.d.ts +28 -7
  48. package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
  49. package/dist/executors/bounded-exec/mbx-pool.js +30 -9
  50. package/dist/executors/bounded-exec/mbx-reservations.d.ts +8 -5
  51. package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
  52. package/dist/executors/bounded-exec/mbx-reservations.js +187 -283
  53. package/dist/executors/bounded-exec/parent-grant.d.ts +2 -0
  54. package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -1
  55. package/dist/executors/bounded-exec/parent-grant.js +1 -0
  56. package/dist/executors/bounded-exec/pool-admission.d.ts +29 -3
  57. package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
  58. package/dist/executors/bounded-exec/pool-admission.js +47 -1
  59. package/dist/executors/bounded-exec/progress.d.ts +3 -0
  60. package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
  61. package/dist/executors/bounded-exec/progress.js +31 -1
  62. package/dist/executors/bounded-exec/reservation-connection.d.ts +90 -0
  63. package/dist/executors/bounded-exec/reservation-connection.d.ts.map +1 -0
  64. package/dist/executors/bounded-exec/reservation-connection.js +248 -0
  65. package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
  66. package/dist/executors/bounded-exec/run-cpu-pool.js +3 -0
  67. package/dist/executors/bounded-exec/runner-sizing.d.ts +9 -5
  68. package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
  69. package/dist/executors/bounded-exec/runner-sizing.js +13 -9
  70. package/dist/executors/bounded-exec/schema.d.ts +2 -2
  71. package/dist/executors/bounded-exec/suite-minimum.d.ts +21 -0
  72. package/dist/executors/bounded-exec/suite-minimum.d.ts.map +1 -0
  73. package/dist/executors/bounded-exec/suite-minimum.js +85 -0
  74. package/dist/executors/bounded-exec/test-result-writer.d.ts +2 -0
  75. package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -1
  76. package/dist/executors/bounded-exec/test-result-writer.js +1 -0
  77. package/dist/executors/bounded-exec/test-results.d.ts +5 -0
  78. package/dist/executors/bounded-exec/test-results.d.ts.map +1 -1
  79. package/dist/executors/bounded-exec/test-results.js +2 -2
  80. package/dist/executors/bounded-exec/verdict.d.ts +26 -4
  81. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  82. package/dist/executors/bounded-exec/verdict.js +26 -5
  83. package/dist/executors/cargo-zigbuild/executor.d.ts +10 -8
  84. package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
  85. package/dist/executors/cargo-zigbuild/executor.js +32 -19
  86. package/dist/index.d.ts.map +1 -1
  87. package/dist/index.js +105 -73
  88. package/dist/managed-files/ci-workflow.d.ts +4 -1
  89. package/dist/managed-files/ci-workflow.d.ts.map +1 -1
  90. package/dist/managed-files/ci-workflow.js +9 -14
  91. package/dist/managed-files/files.d.ts +3 -3
  92. package/dist/managed-files/files.js +3 -3
  93. package/dist/task-results.js +2 -1
  94. package/dist/workspace-config-policy.d.ts.map +1 -1
  95. package/dist/workspace-config-policy.js +135 -42
  96. package/executors.json +2 -0
  97. package/managed/raw/envrc +24 -0
  98. package/managed/raw/patches/nx@24.0.3.patch +1179 -133
  99. package/managed/raw/patches/ttsc@0.37.1.patch +1055 -0
  100. package/managed/raw/tooling/direnv/developer-links.ts +105 -11
  101. package/managed/raw/tooling/direnv/devenv.smoo.nix +20 -19
  102. package/package.json +67 -11
  103. package/runtime/nx-publish/.build-inputs.json +2 -2
  104. package/runtime/nx-publish/cache_adapter.js +293 -3
  105. package/runtime/nx-publish/nx_publish.js +36 -14
  106. package/src/built-output-packages.ts +1 -1
  107. package/src/cargo-closure-input.ts +1 -1
  108. package/src/cargo-source-hash.ts +16 -132
  109. package/src/cargo-toolchain-paths.ts +2 -2
  110. package/src/cargo-workspace.ts +1 -2
  111. package/src/executors/bounded-exec/admission-queue.ts +11 -10
  112. package/src/executors/bounded-exec/awake-clock.ts +20 -0
  113. package/src/executors/bounded-exec/bounds.ts +28 -15
  114. package/src/executors/bounded-exec/case-gate.ts +365 -0
  115. package/src/executors/bounded-exec/cpu-pool-thread.ts +3 -3
  116. package/src/executors/bounded-exec/cpu-pool.ts +139 -12
  117. package/src/executors/bounded-exec/endpoint-life.ts +279 -0
  118. package/src/executors/bounded-exec/executor.ts +280 -106
  119. package/src/executors/bounded-exec/gateway-lease.ts +6 -0
  120. package/src/executors/bounded-exec/go-run-summary.ts +137 -0
  121. package/src/executors/bounded-exec/hang-diagnostics.ts +70 -52
  122. package/src/executors/bounded-exec/lease-wait.ts +1 -0
  123. package/src/executors/bounded-exec/mbx-pool.ts +36 -9
  124. package/src/executors/bounded-exec/mbx-reservations.ts +213 -182
  125. package/src/executors/bounded-exec/parent-grant.ts +3 -0
  126. package/src/executors/bounded-exec/pool-admission.ts +59 -2
  127. package/src/executors/bounded-exec/progress.ts +37 -2
  128. package/src/executors/bounded-exec/reservation-connection.ts +221 -0
  129. package/src/executors/bounded-exec/run-cpu-pool.ts +2 -0
  130. package/src/executors/bounded-exec/runner-sizing.ts +13 -8
  131. package/src/executors/bounded-exec/schema.json +1 -1
  132. package/src/executors/bounded-exec/schema.ts +2 -2
  133. package/src/executors/bounded-exec/suite-minimum.ts +81 -0
  134. package/src/executors/bounded-exec/test-result-writer.ts +3 -0
  135. package/src/executors/bounded-exec/test-results.ts +5 -0
  136. package/src/executors/bounded-exec/verdict.ts +39 -6
  137. package/src/executors/cargo-zigbuild/executor.ts +30 -19
  138. package/src/index.ts +112 -78
  139. package/src/managed-files/ci-workflow.ts +13 -17
  140. package/src/managed-files/files.ts +3 -3
  141. package/src/workspace-config-policy.ts +161 -43
  142. package/test-timings.json +157 -147
  143. package/managed/raw/patches/ttsc@0.36.3.patch +0 -1660
@@ -13,8 +13,9 @@ function openNxCache(root) {
13
13
  // `<machine>-v<DB_VERSION>.db` there (v3 in 23.2.1, v6 in 24.0.3: `native/db/mod.rs`, `initialize.rs`),
14
14
  // reached only through Nx's own `NxCache.get`; an entry is `<cache>/<hash>/` with its terminal output in
15
15
  // `<cache>/terminalOutputs/<hash>` (`native/cache/cache.rs`); `CachedResult` is `{ code, terminalOutput,
16
- // outputsPath, size }`; and `HttpRemoteCache.store` archives the entry alike (`native/cache/http_remote_cache.rs`,
17
- // the same source). 23.2.1's `get` answers a row whose entry directory is gone; 24.0.3's does not.
16
+ // outputsPath, size }`; and `HttpRemoteCache` stores and restores an entry as the archive nxArchive (below) writes
17
+ // (`native/cache/http_remote_cache.rs`, the same source at both tags, built with tar 0.4.44 and flate2 1.1.9 in each
18
+ // tag's Cargo.lock). 23.2.1's `get` answers a row whose entry directory is gone; 24.0.3's does not.
18
19
  const layouts = new Map([
19
20
  ['23.2.1', { workspaceData: '.nx/workspace-data', cache: '.nx/cache' }],
20
21
  ['24.0.3', { workspaceData: '.nx/workspace-data', cache: '.nx/cache' }],
@@ -54,5 +55,294 @@ function openNxCache(root) {
54
55
  if (path.resolve(native.cacheDirectory) !== path.resolve(cacheDir)) {
55
56
  throw new Error(`Nx ${version} cache directory changed: expected ${cacheDir}, actual ${native.cacheDirectory}`);
56
57
  }
57
- return { requireNx, cache, cacheDir: native.cacheDirectory, getLocal: (hash) => native.get(hash) };
58
+ return {
59
+ requireNx,
60
+ cache,
61
+ cacheDir: native.cacheDirectory,
62
+ getLocal: (hash) => native.get(hash),
63
+ archive: (hash, terminalOutput, code) => nxArchive(path.join(native.cacheDirectory, hash), terminalOutput, code),
64
+ };
65
+ }
66
+
67
+ // The archive of `<cache>/<hash>` Nx's native `HttpRemoteCache.store` sends and its `retrieve` unpacks in any checkout,
68
+ // written here so a land archives only a key its store lacks, and at gzip level 1 rather than Nx's default 6 (retrieve
69
+ // gunzips any level). With tar 0.4.44's `Builder::new()` (complete header mode) and symlinks not followed, `store` writes:
70
+ // - `append_dir_all("", <cache>/<hash>)`: a stack walk from the entry directory, which pushes each directory's
71
+ // `read_dir` entries in readdir order and takes the last pushed first, writing a directory before its children. Each
72
+ // entry has a GNU header (magic "ustar ", version " \0") filled from its metadata: the mtime in whole seconds, uid,
73
+ // gid, the whole st_mode, the size (0 for a directory or symlink), device fields of zero, and a symlink's target. A
74
+ // path or target over 100 bytes is first written whole and NUL-terminated in a `././@LongLink` entry ('L' or 'K');
75
+ // - `terminalOutput` (its UTF-8 bytes) and `code` (four big-endian bytes) under old-style headers;
76
+ // - two zero blocks.
77
+ // Numbers are tar 0.4.44's: octal, NUL-terminated and zero-padded, or big-endian binary with the high bit set past what
78
+ // the field holds. These are Nx's own bytes (the publisher's tests compare them), except that a sparse file, which tar
79
+ // 0.4.44 records as GNU sparse on Linux only, is written whole; `retrieve` restores the same file either way.
80
+ const TAR_BLOCK = 512;
81
+ /** gzip is fed writes of at least 1 MiB, not one per 512-byte header. */
82
+ const TAR_WRITE = 1 << 20;
83
+ const TAR_TYPES = new Map([
84
+ [0o100000n, '0'],
85
+ [0o120000n, '2'],
86
+ [0o040000n, '5'],
87
+ ]);
88
+
89
+ /**
90
+ * Nx's archive of the entry directory `entry` (above), gzipped at level 1: the body of one PUT as its compressed chunks
91
+ * and their total length, sent as they are rather than joined into a second copy.
92
+ */
93
+ async function nxArchive(entry, terminalOutput, code) {
94
+ const { createGzip } = require('node:zlib');
95
+ const { pipeline } = require('node:stream/promises');
96
+ const compressed = [];
97
+ await pipeline(tarWrites(tarStream(entry, terminalOutput, code)), createGzip({ level: 1 }), async (gzip) => {
98
+ for await (const chunk of gzip) compressed.push(chunk);
99
+ });
100
+ return { chunks: compressed, length: compressed.reduce((sum, chunk) => sum + chunk.length, 0) };
101
+ }
102
+
103
+ /** Joins successive byte strings into writes of at least TAR_WRITE bytes. */
104
+ function* tarWrites(parts) {
105
+ let held = [];
106
+ let size = 0;
107
+ for (const part of parts) {
108
+ held.push(part);
109
+ size += part.length;
110
+ if (size >= TAR_WRITE) {
111
+ yield held.length === 1 ? held[0] : Buffer.concat(held, size);
112
+ held = [];
113
+ size = 0;
114
+ }
115
+ }
116
+ if (size > 0) yield Buffer.concat(held, size);
117
+ }
118
+
119
+ /** The tar stream `HttpRemoteCache.store` builds for `entry`, as successive byte strings. */
120
+ function* tarStream(entry, terminalOutput, code) {
121
+ const fs = require('node:fs');
122
+ // append_dir_all's stack: [source path, archive path (null for the entry itself), is a directory].
123
+ const stack = [[Buffer.from(entry), null, true]];
124
+ while (stack.length > 0) {
125
+ const [source, name, isDirectory] = stack.pop();
126
+ if (isDirectory) {
127
+ const directory = fs.opendirSync(source, { encoding: 'buffer' });
128
+ try {
129
+ for (let child = directory.readSync(); child !== null; child = directory.readSync()) {
130
+ const archived = name === null ? child.name : Buffer.concat([name, TAR_SLASH, child.name]);
131
+ stack.push([Buffer.concat([source, TAR_SLASH, child.name]), archived, child.isDirectory()]);
132
+ }
133
+ } finally {
134
+ directory.closeSync();
135
+ }
136
+ if (name !== null) yield* tarEntry(name, fs.statSync(source, { bigint: true }), null);
137
+ continue;
138
+ }
139
+ const stat = fs.lstatSync(source, { bigint: true });
140
+ if (stat.isSymbolicLink()) {
141
+ yield* tarEntry(name, stat, fs.readlinkSync(source, { encoding: 'buffer' }));
142
+ } else if (stat.isFile()) {
143
+ yield* tarFile(fs, source, name);
144
+ } else {
145
+ throw new Error(`${source} is neither a directory, a file nor a symlink, which no Nx cache entry holds`);
146
+ }
147
+ }
148
+ yield* tarOld('terminalOutput', Buffer.from(terminalOutput, 'utf8'));
149
+ const exitCode = Buffer.alloc(4);
150
+ exitCode.writeUInt32BE(code);
151
+ yield* tarOld('code', exitCode);
152
+ yield Buffer.alloc(2 * TAR_BLOCK);
153
+ }
154
+
155
+ /** A regular file's header and bytes, its length the one its header names: a file that changes meanwhile is refused. */
156
+ function* tarFile(fs, source, name) {
157
+ const fd = fs.openSync(source, 'r');
158
+ try {
159
+ const stat = fs.fstatSync(fd, { bigint: true });
160
+ yield* tarEntry(name, stat, null);
161
+ let left = Number(stat.size);
162
+ while (left > 0) {
163
+ const chunk = Buffer.allocUnsafe(Math.min(left, TAR_WRITE));
164
+ const read = fs.readSync(fd, chunk, 0, chunk.length, null);
165
+ if (read === 0) throw new Error(`${source} shrank while it was archived`);
166
+ left -= read;
167
+ yield read === chunk.length ? chunk : chunk.subarray(0, read);
168
+ }
169
+ if (fs.readSync(fd, Buffer.alloc(1), 0, 1, null) !== 0) throw new Error(`${source} grew while it was archived`);
170
+ yield* tarPadding(stat.size);
171
+ } finally {
172
+ fs.closeSync(fd);
173
+ }
174
+ }
175
+
176
+ /** tar 0.4.44's `append_fs`/`append_file` header for one walked entry, after any long-name entries it needs. */
177
+ function* tarEntry(name, stat, link) {
178
+ const header = tarGnuHeader();
179
+ yield* tarPath(header, name);
180
+ tarNumber(header, 136, 12, tarSeconds(stat.mtimeNs));
181
+ tarNumber(header, 108, 8, stat.uid);
182
+ tarNumber(header, 116, 8, stat.gid);
183
+ tarOctal(header, 100, 8, stat.mode);
184
+ header.write(TAR_TYPES.get(stat.mode & 0o170000n), 156, 'latin1');
185
+ tarNumber(header, 124, 12, stat.isFile() ? stat.size : 0n);
186
+ tarOctal(header, 329, 8, 0n);
187
+ tarOctal(header, 337, 8, 0n);
188
+ if (link !== null) yield* tarLink(header, link);
189
+ tarChecksum(header);
190
+ yield header;
191
+ }
192
+
193
+ /** `append_data` under `Header::new_old()`: no magic, mode, owner or type, only the name, size and a zero mtime. */
194
+ function* tarOld(name, data) {
195
+ const header = Buffer.alloc(TAR_BLOCK);
196
+ tarNumber(header, 136, 12, 0n);
197
+ tarNumber(header, 124, 12, BigInt(data.length));
198
+ tarCopyPath(header, 0, 100, Buffer.from(name), false, false);
199
+ tarChecksum(header);
200
+ yield header;
201
+ if (data.length > 0) yield data;
202
+ yield* tarPadding(BigInt(data.length));
203
+ }
204
+
205
+ function tarGnuHeader() {
206
+ const header = Buffer.alloc(TAR_BLOCK);
207
+ header.write('ustar ', 257, 'latin1');
208
+ header.write(' \0', 263, 'latin1');
209
+ tarNumber(header, 136, 12, 0n);
210
+ return header;
211
+ }
212
+
213
+ /** `prepare_header_path`: the name field, or a GNU long-name entry and the name cut to its valid UTF-8 within 100 bytes. */
214
+ function* tarPath(header, name) {
215
+ try {
216
+ tarCopyPath(header, 0, 100, name, false, false);
217
+ } catch (error) {
218
+ if (name.length < 100) throw error;
219
+ yield* tarLongLink(name, 'L');
220
+ const { isUtf8 } = require('node:buffer');
221
+ let cut = 100;
222
+ while (cut > 0 && !isUtf8(name.subarray(0, cut))) cut -= 1;
223
+ tarCopyPath(header, 0, 100, name.subarray(0, cut), false, true);
224
+ }
225
+ }
226
+
227
+ /** `prepare_header_link`: the link-name field, or a GNU long-link entry before the header. */
228
+ function* tarLink(header, link) {
229
+ try {
230
+ tarCopyPath(header, 157, 100, link, true, false);
231
+ } catch (error) {
232
+ if (link.length < 100) throw error;
233
+ yield* tarLongLink(link, 'K');
234
+ }
235
+ }
236
+
237
+ /** tar 0.4.44's `prepare_header`: a `././@LongLink` entry holding `data` NUL-terminated. */
238
+ function* tarLongLink(data, type) {
239
+ const header = tarGnuHeader();
240
+ header.write('././@LongLink', 0, 'latin1');
241
+ tarOctal(header, 100, 8, 0o644n);
242
+ tarNumber(header, 108, 8, 0n);
243
+ tarNumber(header, 116, 8, 0n);
244
+ tarNumber(header, 124, 12, BigInt(data.length + 1));
245
+ header.write(type, 156, 'latin1');
246
+ tarChecksum(header);
247
+ yield header;
248
+ yield Buffer.concat([data, Buffer.alloc(1)]);
249
+ yield* tarPadding(BigInt(data.length + 1));
250
+ }
251
+
252
+ const TAR_SLASH = Buffer.from('/');
253
+
254
+ /**
255
+ * tar 0.4.44's `copy_path_into_inner`: `path`'s components (std's `Path::components`) joined by '/' into the field,
256
+ * each copy NUL-terminated while room remains. Like it, a component that does not fit throws after the ones before it
257
+ * were written, and those stay in the header.
258
+ */
259
+ function tarCopyPath(header, offset, length, path, isLinkName, isTruncatedGnuLongPath) {
260
+ const end = offset + length;
261
+ let at = offset;
262
+ const copy = (bytes) => {
263
+ if (bytes.length > end - at) throw new Error(`${path} is too long for a ${length}-byte tar field`);
264
+ if (bytes.includes(0)) throw new Error(`${path} contains a NUL byte`);
265
+ bytes.copy(header, at);
266
+ if (at + bytes.length < end) header[at + bytes.length] = 0;
267
+ at += bytes.length;
268
+ };
269
+ const components = tarComponents(path);
270
+ let needsSlash = false;
271
+ let emitted = false;
272
+ for (const [index, { kind, bytes }] of components.entries()) {
273
+ if (!isLinkName) {
274
+ if (kind === 'root') throw new Error(`${path}: paths in archives must be relative`);
275
+ if (kind === 'parent' && (!isTruncatedGnuLongPath || index < components.length - 1)) {
276
+ throw new Error(`${path}: paths in archives must not have '..'`);
277
+ }
278
+ if (kind === 'current' && components.length !== 1) continue;
279
+ }
280
+ if (needsSlash) copy(TAR_SLASH);
281
+ copy(bytes);
282
+ if (kind !== 'root') needsSlash = true;
283
+ emitted = true;
284
+ }
285
+ if (!emitted) throw new Error(`${path}: paths in archives must have at least one component`);
286
+ if (path.length > 0 && path[path.length - 1] === 0x2f) copy(TAR_SLASH);
287
+ }
288
+
289
+ /** std's Unix `Path::components`: a leading root, a leading '.', then the '/'-separated names past empty and '.' ones. */
290
+ function tarComponents(path) {
291
+ const components = [];
292
+ if (path[0] === 0x2f) components.push({ kind: 'root', bytes: TAR_SLASH });
293
+ else if (path[0] === 0x2e && (path.length === 1 || path[1] === 0x2f)) {
294
+ components.push({ kind: 'current', bytes: Buffer.from('.') });
295
+ }
296
+ let start = 0;
297
+ for (let index = 0; index <= path.length; index++) {
298
+ if (index < path.length && path[index] !== 0x2f) continue;
299
+ const bytes = path.subarray(start, index);
300
+ start = index + 1;
301
+ if (bytes.length === 0 || (bytes.length === 1 && bytes[0] === 0x2e)) continue;
302
+ const parent = bytes.length === 2 && bytes[0] === 0x2e && bytes[1] === 0x2e;
303
+ components.push({ kind: parent ? 'parent' : 'normal', bytes });
304
+ }
305
+ return components;
306
+ }
307
+
308
+ /** tar 0.4.44's `octal_into`: the low digits that fit, zero-padded, NUL last. */
309
+ function tarOctal(header, offset, length, value) {
310
+ const digits = value.toString(8);
311
+ header[offset + length - 1] = 0;
312
+ for (let place = 1; place < length; place++) {
313
+ const digit = digits.length - place;
314
+ header[offset + length - 1 - place] = digit >= 0 ? digits.charCodeAt(digit) : 0x30;
315
+ }
316
+ }
317
+
318
+ /** tar 0.4.44's `num_field_wrapper_into`: octal, or big-endian binary with the high bit set past 2^33 (2^21 in 8 bytes). */
319
+ function tarNumber(header, offset, length, value) {
320
+ if (value >= 1n << 33n || (value >= 1n << 21n && length === 8)) {
321
+ header.fill(0, offset, offset + length - 8);
322
+ header.writeBigUInt64BE(value, offset + length - 8);
323
+ header[offset] |= 0x80;
324
+ } else {
325
+ tarOctal(header, offset, length, value);
326
+ }
327
+ }
328
+
329
+ /** `set_cksum`: the byte sum with the checksum field read as spaces, in its octal form. */
330
+ function tarChecksum(header) {
331
+ let sum = 8 * 0x20;
332
+ for (let index = 0; index < TAR_BLOCK; index++) {
333
+ if (index < 148 || index >= 156) sum += header[index];
334
+ }
335
+ tarOctal(header, 148, 8, BigInt(sum));
336
+ }
337
+
338
+ /** st_mtime as `MetadataExt::mtime() as u64`: whole seconds, floored, two's complement when negative. */
339
+ function tarSeconds(nanoseconds) {
340
+ const seconds = nanoseconds / 1_000_000_000n;
341
+ const floored = nanoseconds < 0n && seconds * 1_000_000_000n !== nanoseconds ? seconds - 1n : seconds;
342
+ return BigInt.asUintN(64, floored);
343
+ }
344
+
345
+ function* tarPadding(size) {
346
+ const rest = Number(size % BigInt(TAR_BLOCK));
347
+ if (rest !== 0) yield Buffer.alloc(TAR_BLOCK - rest);
58
348
  }
@@ -1,6 +1,8 @@
1
- // A land's publish (05_gateway.md "`/nx/v1/cache/{hash}`"): after the gate passed, every task result
2
- // the gate's Nx runs named is stored to the project's Nx cache from this checkout's own cache, as
3
- // plain PUTs by the stock native client. Prefixed once by cache_adapter::script.
1
+ // A land's publish (05_gateway.md, "Publication"; 03_caches.md, "Nx accepted-task artifact service"): after the gate
2
+ // passed, every task result the gate's Nx runs named is stored to the project's Nx cache from this checkout's own cache.
3
+ // Each key is asked for first: a stored key is counted and never archived. Only a missing key is archived, by
4
+ // cache_adapter's nxArchive in Nx's own layout at gzip level 1, and sent in one plain PUT. Prefixed once by
5
+ // cache_adapter::script.
4
6
  // Usage: node - <absolute Nx root> <JSON array of task hashes>; answers one JSON line on stdout.
5
7
  const answer = process.stdout.write.bind(process.stdout);
6
8
  process.stdout.write = process.stderr.write.bind(process.stderr);
@@ -22,11 +24,12 @@ for (const hash of hashes) {
22
24
  }
23
25
  const server = process.env.NX_SELF_HOSTED_REMOTE_CACHE_SERVER;
24
26
  if (!server) throw new Error('NX_SELF_HOSTED_REMOTE_CACHE_SERVER names no project Nx cache');
27
+ // The headers Nx's own client sends (`HttpRemoteCache::new`): the configured bearer, and an octet-stream body.
28
+ const token = process.env.NX_SELF_HOSTED_REMOTE_CACHE_ACCESS_TOKEN;
29
+ const bearer = token === undefined ? {} : { authorization: `Bearer ${token}` };
25
30
 
26
31
  async function publish() {
27
- const { requireNx, cacheDir, getLocal } = openNxCache(root);
28
- const { HttpRemoteCache } = requireNx('nx/src/native');
29
- const remote = new HttpRemoteCache();
32
+ const { getLocal, archive } = openNxCache(root);
30
33
  let published = 0;
31
34
  let existing = 0;
32
35
  let skipped = 0;
@@ -38,17 +41,36 @@ async function publish() {
38
41
  skipped += 1;
39
42
  continue;
40
43
  }
41
- if (await remote.store(hash, cacheDir, result.terminalOutput ?? '', 0)) {
42
- published += 1;
44
+ const url = `${server}/v1/cache/${hash}`;
45
+ const probe = await fetch(url, { headers: { ...bearer, accept: 'application/octet-stream' } });
46
+ await probe.body?.cancel();
47
+ if (probe.status === 200) {
48
+ existing += 1;
43
49
  continue;
44
50
  }
45
- // The native store answers false for an existing key (409) and for a refusal: tell them apart.
46
- const response = await fetch(`${server}/v1/cache/${hash}`);
47
- await response.body?.cancel();
48
- if (response.status !== 200) {
49
- throw new Error(`task ${hash} was not stored, and the project cache answers GET ${response.status}`);
51
+ if (probe.status !== 404) {
52
+ throw new Error(`task ${hash} was not stored: the project cache answers GET ${probe.status}`);
53
+ }
54
+ const { chunks, length } = await archive(hash, result.terminalOutput ?? '', 0);
55
+ const response = await fetch(url, {
56
+ method: 'PUT',
57
+ headers: { ...bearer, 'content-type': 'application/octet-stream', 'content-length': String(length) },
58
+ body: (async function* () {
59
+ yield* chunks;
60
+ })(),
61
+ duplex: 'half',
62
+ });
63
+ const said = (await response.text()).trim();
64
+ if (response.status === 200) {
65
+ published += 1;
66
+ } else if (response.status === 409) {
67
+ // Stored between the GET and this PUT: the store keeps the archive it holds.
68
+ existing += 1;
69
+ } else {
70
+ throw new Error(
71
+ `task ${hash} was not stored: the project cache answers PUT ${response.status}${said === '' ? '' : `: ${said}`}`,
72
+ );
50
73
  }
51
- existing += 1;
52
74
  }
53
75
  return { published, existing, skipped };
54
76
  }
@@ -72,7 +72,7 @@ function readLockfile(context: CreateNodesContextV2): LockfileRead {
72
72
  if (!existsSync(path)) throw new Error(`builtOutputPackages names packages, but ${path} does not exist`);
73
73
  const contents = packageManager === 'bun' ? readBunLockFile(path) : readFileSync(path, 'utf-8');
74
74
  const hash = hashArray([nxVersion, contents]);
75
- if (published?.hash === hash) return published;
75
+ if (packageManager !== 'bun' && published?.hash === hash) return published;
76
76
  const { nodes, keyMap } = getLockFileNodes(packageManager, lockFile, contents, hash, context);
77
77
  return { packageManager, lockFile, contents, hash, nodes, keyMap };
78
78
  }
@@ -306,7 +306,7 @@ export async function cargoLockNode(
306
306
  const paths = new Map(cargo.local.map((pkg) => [pkg.id, `path:${pkg.name}`]));
307
307
  const label = (id: string) => paths.get(id) ?? id;
308
308
  const rows = [...closure]
309
- .map(([id, built]) => [label(id), cargo.locked.get(id) ?? '', ...built.map(label).sort()].join('\t'))
309
+ .map(([id, built]) => [label(id), cargo.pinned.get(id)?.source ?? '', ...built.map(label).sort()].join('\t'))
310
310
  .sort();
311
311
  hash.update(`${source.manifest}\n${rows.join('\n')}\n\0`);
312
312
  }
@@ -16,6 +16,17 @@ import { createRequire } from 'node:module';
16
16
  import { homedir } from 'node:os';
17
17
  import { basename, delimiter, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
18
18
  import type { Readable } from 'node:stream';
19
+ import {
20
+ buildClosure,
21
+ type CargoClosure,
22
+ type CargoEdge,
23
+ type CargoMetadata,
24
+ cargoEdges,
25
+ localPackagesUnder,
26
+ type PinnedCargoPackage,
27
+ parseCargoMetadata,
28
+ pinnedPackages,
29
+ } from '@smoothbricks/build-inputs/cargo-closure';
19
30
  import { CARGO_ANCESTOR_INPUTS } from '@smoothbricks/cargo-retain/cargo-ancestor-inputs';
20
31
  import native, { type FileData } from 'nx/src/native/index.js';
21
32
  import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
@@ -27,99 +38,6 @@ const { IS_WASM, WorkspaceContext, hashArray } = native;
27
38
 
28
39
  const cargoResolveEvents = channel('@smoothbricks/nx-plugin:cargo-resolve');
29
40
 
30
- /** The fields of `cargo metadata --format-version 1` this module reads. */
31
- interface CargoMetadata {
32
- packages: {
33
- id: string;
34
- name: string;
35
- version: string;
36
- source: string | null;
37
- manifest_path: string;
38
- targets: { src_path: string }[];
39
- /** Added by the cache after Cargo identifies each local package's governing workspace. */
40
- governing_manifest_path?: string;
41
- }[];
42
- resolve: { nodes: { id: string; deps: { pkg: string; dep_kinds: { kind: CargoDependencyKind }[] }[] }[] } | null;
43
- workspace_members: string[];
44
- workspace_root: string;
45
- }
46
-
47
- /**
48
- * Narrowed field by field, not by typia: graph inference imports this module,
49
- * and a workspace that loads the plugin from its sources runs them without
50
- * typia's compile-time transform.
51
- */
52
- function parseCargoMetadata(json: string): CargoMetadata {
53
- const member = (value: unknown, key: string): unknown =>
54
- typeof value === 'object' && value !== null && key in value ? Reflect.get(value, key) : undefined;
55
- const text = (value: unknown, field: string): string => {
56
- if (typeof value !== 'string') throw new Error(`cargo metadata reported no valid ${field}`);
57
- return value;
58
- };
59
- const list = (value: unknown, field: string): unknown[] => {
60
- if (!Array.isArray(value)) throw new Error(`cargo metadata reported no valid ${field}`);
61
- return value;
62
- };
63
- const document: unknown = JSON.parse(json);
64
- const resolveGraph = member(document, 'resolve');
65
- return {
66
- packages: list(member(document, 'packages'), 'packages').map((pkg) => ({
67
- id: text(member(pkg, 'id'), 'package id'),
68
- name: text(member(pkg, 'name'), 'package name'),
69
- version: text(member(pkg, 'version'), 'package version'),
70
- source: member(pkg, 'source') === null ? null : text(member(pkg, 'source'), 'package source'),
71
- manifest_path: text(member(pkg, 'manifest_path'), 'package manifest_path'),
72
- targets: list(member(pkg, 'targets'), 'package targets').map((target) => ({
73
- src_path: text(member(target, 'src_path'), 'target src_path'),
74
- })),
75
- ...(member(pkg, 'governing_manifest_path') === undefined
76
- ? {}
77
- : { governing_manifest_path: text(member(pkg, 'governing_manifest_path'), 'governing manifest') }),
78
- })),
79
- resolve:
80
- resolveGraph === null
81
- ? null
82
- : {
83
- nodes: list(member(resolveGraph, 'nodes'), 'resolve').map((node) => ({
84
- id: text(member(node, 'id'), 'resolve node id'),
85
- deps: list(member(node, 'deps'), 'resolve node deps').map((dependency) => ({
86
- pkg: text(member(dependency, 'pkg'), 'resolve edge'),
87
- dep_kinds: dependencyKinds(member(dependency, 'dep_kinds')),
88
- })),
89
- })),
90
- },
91
- workspace_members: list(member(document, 'workspace_members'), 'workspace_members').map((id) =>
92
- text(id, 'workspace member'),
93
- ),
94
- workspace_root: text(member(document, 'workspace_root'), 'workspace_root'),
95
- };
96
- }
97
-
98
- /** How a package declares a dependency: `[dependencies]` (null), `[dev-dependencies]` or `[build-dependencies]`. */
99
- type CargoDependencyKind = null | 'dev' | 'build';
100
-
101
- /** A resolve edge's `dep_kinds`. An edge Cargo reports with no kind is refused: read as dev-only, it would drop a dependency. */
102
- function dependencyKinds(value: unknown): { kind: CargoDependencyKind }[] {
103
- if (!Array.isArray(value) || value.length === 0)
104
- throw new Error('cargo metadata reported no valid resolve edge kinds');
105
- return value.map((entry: unknown) => {
106
- const kind = typeof entry === 'object' && entry !== null ? Reflect.get(entry, 'kind') : undefined;
107
- if (kind !== null && kind !== 'dev' && kind !== 'build')
108
- throw new Error('cargo metadata reported no valid resolve edge kind');
109
- return { kind };
110
- });
111
- }
112
-
113
- /** One resolved dependency of a package: the package it resolved to, and whether every declaration of it is a dev one. */
114
- export interface CargoEdge {
115
- readonly pkg: string;
116
- /** Cargo builds this dependency for the dependent's tests, examples and benches only. */
117
- readonly devOnly: boolean;
118
- }
119
-
120
- /** Every package Cargo builds for a selection, by package id, each with the dependencies Cargo builds for it. */
121
- export type CargoClosure = ReadonlyMap<string, readonly string[]>;
122
-
123
41
  /** A mutable (path) package in Cargo's locked resolve. Every path is canonical. */
124
42
  export interface LocalCargoPackage {
125
43
  readonly id: string;
@@ -139,11 +57,8 @@ export interface CargoResolve {
139
57
  /** The workspace's own members: `root`'s manifest governs each of them. */
140
58
  readonly members: ReadonlySet<string>;
141
59
  readonly local: readonly LocalCargoPackage[];
142
- /**
143
- * Every package Cargo.lock pins (registry, git, alternate registry), by package id: its `source`, which for a git
144
- * dependency names the locked commit that the id alone does not.
145
- */
146
- readonly locked: ReadonlyMap<string, string>;
60
+ /** Every package Cargo.lock pins (registry, git, alternate registry), by package id. */
61
+ readonly pinned: ReadonlyMap<string, PinnedCargoPackage>;
147
62
  /** Every dependency edge by package id, or null when Cargo reported no resolution. */
148
63
  readonly edges: ReadonlyMap<string, readonly CargoEdge[]> | null;
149
64
  }
@@ -534,16 +449,8 @@ function cargoFromMetadata(metadata: CargoMetadata): CargoResolve {
534
449
  governingManifest: pkg.governing_manifest_path,
535
450
  };
536
451
  }),
537
- locked: new Map(metadata.packages.flatMap((pkg) => (pkg.source === null ? [] : [[pkg.id, pkg.source] as const]))),
538
- edges:
539
- metadata.resolve === null
540
- ? null
541
- : new Map(
542
- metadata.resolve.nodes.map((node) => [
543
- node.id,
544
- node.deps.map((dep) => ({ pkg: dep.pkg, devOnly: dep.dep_kinds.every(({ kind }) => kind === 'dev') })),
545
- ]),
546
- ),
452
+ pinned: pinnedPackages(metadata),
453
+ edges: cargoEdges(metadata),
547
454
  };
548
455
  }
549
456
 
@@ -1073,34 +980,11 @@ export async function hashCargoPathInputs(
1073
980
  */
1074
981
  export async function dependencyClosure(cargo: CargoResolve, directory: string): Promise<CargoClosure> {
1075
982
  const base = await realpath(directory);
1076
- const selected = cargo.local
1077
- .filter((pkg) => pkg.directory === base || pkg.directory.startsWith(`${base}${sep}`))
1078
- .map((pkg) => pkg.id);
983
+ const selected = localPackagesUnder(cargo.local, base).map((pkg) => pkg.id);
1079
984
  if (selected.length === 0) throw new Error(`no local Cargo package lies under ${directory}`);
1080
985
  return buildClosure(cargo, selected);
1081
986
  }
1082
987
 
1083
- /**
1084
- * The packages Cargo compiles to build, test or run the `selected` ones, each with the dependencies Cargo builds for
1085
- * it. A selected package's targets reach every dependency kind: its library and binaries the normal ones, its build
1086
- * script the build ones, its tests, examples and benches the dev ones. Below the selection Cargo builds a package for
1087
- * its library and build script alone, so a dependency's own dev-dependencies never compile into a selected package's
1088
- * binaries and are not part of what it builds.
1089
- */
1090
- export function buildClosure(cargo: CargoResolve, selected: readonly string[]): CargoClosure {
1091
- if (cargo.edges === null) throw new Error('cargo metadata reported no dependency resolution');
1092
- const roots = new Set(selected);
1093
- const reached = new Map<string, readonly string[]>();
1094
- const pending = [...selected];
1095
- for (let id = pending.pop(); id !== undefined; id = pending.pop()) {
1096
- if (reached.has(id)) continue;
1097
- const built = (cargo.edges.get(id) ?? []).filter((edge) => roots.has(id) || !edge.devOnly).map((edge) => edge.pkg);
1098
- reached.set(id, built);
1099
- pending.push(...built);
1100
- }
1101
- return reached;
1102
- }
1103
-
1104
988
  /**
1105
989
  * Whether git ignores a path beneath one of the package directories. A
1106
990
  * package whose own directory is ignored (an installed dependency, say) keeps
@@ -19,8 +19,8 @@ import { delimiter, join } from 'node:path';
19
19
  * wrapper's object, and one entry covers all three.
20
20
  *
21
21
  * rustc and cargo are not here on purpose. Cargo keys units on the output of `rustc -vV`, never on rustc's path, and the
22
- * version recipe beside this one hashes it. On a non-Linux host the default shell and its linux-cross profile carry
23
- * different rust-std targets, so a rustc path would split one target's key between the two shells.
22
+ * version recipe beside this one hashes it. Shells of one host may carry different rust-std targets, so a rustc path
23
+ * would split one target's key between them.
24
24
  *
25
25
  * Only a path in the Nix store counts. A store object is named by the hash of everything it was built from, so one
26
26
  * shell gives one object on every machine of a platform. A path elsewhere belongs to the machine and takes no part.
@@ -1,11 +1,10 @@
1
1
  import { existsSync, readdirSync, readFileSync } from 'node:fs';
2
2
  import { readFile, realpath } from 'node:fs/promises';
3
3
  import { basename, dirname, isAbsolute, join, posix, relative, resolve, sep } from 'node:path';
4
+ import { buildClosure } from '@smoothbricks/build-inputs/cargo-closure';
4
5
  import { readNxJson } from 'nx/src/config/nx-json.js';
5
6
  import { parse as parseToml, type TomlTable } from 'smol-toml';
6
-
7
7
  import {
8
- buildClosure,
9
8
  type CargoResolve,
10
9
  type CargoSourceIndex,
11
10
  type CompiledCargoInput,