nx 23.3.0 → 24.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +3 -6
  2. package/dist/nx-cloud-internals.d.ts +1 -0
  3. package/dist/nx-cloud-internals.js +7 -1
  4. package/dist/src/adapter/ngcli-adapter.js +1 -0
  5. package/dist/src/command-line/configure-ai-agents/configure-ai-agents.js +0 -12
  6. package/dist/src/command-line/graph/graph.js +3 -5
  7. package/dist/src/command-line/init/command-object.js +0 -12
  8. package/dist/src/command-line/init/init-v2.js +0 -2
  9. package/dist/src/command-line/migrate/agentic/close-agent-session.d.ts +1 -1
  10. package/dist/src/command-line/migrate/agentic/close-agent-session.js +4 -3
  11. package/dist/src/command-line/migrate/agentic/master/spawn-master.js +76 -15
  12. package/dist/src/command-line/migrate/execute-migration.d.ts +2 -2
  13. package/dist/src/command-line/migrate/execute-migration.js +18 -10
  14. package/dist/src/command-line/migrate/migrate-commits.d.ts +4 -1
  15. package/dist/src/command-line/migrate/migrate-commits.js +10 -3
  16. package/dist/src/command-line/migrate/migrate.js +43 -6
  17. package/dist/src/command-line/migrate/run/agent-output.d.ts +1 -0
  18. package/dist/src/command-line/migrate/run/agent-output.js +5 -0
  19. package/dist/src/command-line/migrate/run/broker.d.ts +14 -5
  20. package/dist/src/command-line/migrate/run/broker.js +68 -20
  21. package/dist/src/command-line/migrate/run/index.d.ts +4 -3
  22. package/dist/src/command-line/migrate/run/index.js +8 -1
  23. package/dist/src/command-line/migrate/run/orchestrator.js +72 -13
  24. package/dist/src/command-line/migrate/run/runbook.js +19 -0
  25. package/dist/src/command-line/migrate/run/state-machine.js +4 -1
  26. package/dist/src/command-line/migrate/run/util.d.ts +3 -0
  27. package/dist/src/command-line/migrate/run/util.js +7 -1
  28. package/dist/src/command-line/nx-cloud/connect/connect-to-nx-cloud.js +17 -12
  29. package/dist/src/command-line/release/utils/remote-release-clients/github.js +5 -6
  30. package/dist/src/command-line/release/utils/remote-release-clients/gitlab.js +5 -6
  31. package/dist/src/command-line/run/executor-utils.js +1 -0
  32. package/dist/src/command-line/run/run-one.d.ts +20 -1
  33. package/dist/src/command-line/run/run-one.js +153 -5
  34. package/dist/src/command-line/show/show-target/info.js +1 -1
  35. package/dist/src/command-line/show/show-target/utils.js +1 -0
  36. package/dist/src/command-line/watch/command-object.js +3 -22
  37. package/dist/src/command-line/watch/watch.d.ts +0 -7
  38. package/dist/src/command-line/watch/watch.js +1 -1
  39. package/dist/src/config/misc-interfaces.d.ts +12 -0
  40. package/dist/src/config/task-graph.d.ts +1 -1
  41. package/dist/src/config/workspace-json-project-json.d.ts +65 -0
  42. package/dist/src/core/graph/main.js +1 -1
  43. package/dist/src/daemon/client/client.d.ts +9 -0
  44. package/dist/src/daemon/client/client.js +89 -37
  45. package/dist/src/daemon/server/shutdown-utils.js +0 -3
  46. package/dist/src/daemon/socket-utils.d.ts +9 -4
  47. package/dist/src/daemon/socket-utils.js +12 -49
  48. package/dist/src/daemon/tmp-dir.d.ts +19 -14
  49. package/dist/src/daemon/tmp-dir.js +142 -354
  50. package/dist/src/devkit-exports.d.ts +1 -1
  51. package/dist/src/devkit-internals.d.ts +1 -1
  52. package/dist/src/devkit-internals.js +1 -0
  53. package/dist/src/hasher/check-task-files.js +4 -4
  54. package/dist/src/hasher/file-hasher.js +9 -7
  55. package/dist/src/hasher/hash-task.js +4 -0
  56. package/dist/src/hasher/native-task-hasher-impl.js +3 -1
  57. package/dist/src/migrations/update-24-0-0/find-custom-hashers.d.ts +3 -0
  58. package/dist/src/migrations/update-24-0-0/find-custom-hashers.js +100 -0
  59. package/dist/src/migrations/update-24-0-0/find-custom-hashers.md +63 -0
  60. package/dist/src/migrations/update-24-0-0/move-custom-hashers-to-inputs.md +145 -0
  61. package/dist/src/native/index.d.ts +180 -12
  62. package/dist/src/native/native-bindings.js +8 -0
  63. package/dist/src/native/nx.wasi-browser.js +1 -0
  64. package/dist/src/native/nx.wasi.cjs +1 -0
  65. package/dist/src/native/nx.wasm32-wasi.debug.wasm +0 -0
  66. package/dist/src/native/nx.wasm32-wasi.wasm +0 -0
  67. package/dist/src/nx-cloud/update-manager.d.ts +7 -0
  68. package/dist/src/nx-cloud/update-manager.js +237 -50
  69. package/dist/src/project-graph/affected/affected-tasks.js +1 -0
  70. package/dist/src/project-graph/plugins/isolation/isolated-plugin.js +7 -0
  71. package/dist/src/tasks-runner/create-task-graph.d.ts +8 -7
  72. package/dist/src/tasks-runner/create-task-graph.js +47 -26
  73. package/dist/src/tasks-runner/init-tasks-runner.js +13 -0
  74. package/dist/src/tasks-runner/life-cycle.d.ts +4 -1
  75. package/dist/src/tasks-runner/life-cycle.js +7 -0
  76. package/dist/src/tasks-runner/life-cycles/performance-analysis.d.ts +5 -2
  77. package/dist/src/tasks-runner/life-cycles/performance-analysis.js +13 -7
  78. package/dist/src/tasks-runner/life-cycles/performance-life-cycle.d.ts +4 -3
  79. package/dist/src/tasks-runner/life-cycles/performance-life-cycle.js +9 -2
  80. package/dist/src/tasks-runner/readiness/probes.d.ts +9 -0
  81. package/dist/src/tasks-runner/readiness/probes.js +62 -0
  82. package/dist/src/tasks-runner/readiness/ready-when.d.ts +38 -0
  83. package/dist/src/tasks-runner/readiness/ready-when.js +190 -0
  84. package/dist/src/tasks-runner/run-command.js +5 -4
  85. package/dist/src/tasks-runner/running-tasks/node-child-process.js +5 -4
  86. package/dist/src/tasks-runner/task-env.d.ts +10 -0
  87. package/dist/src/tasks-runner/task-env.js +26 -5
  88. package/dist/src/tasks-runner/task-graph-utils.d.ts +9 -6
  89. package/dist/src/tasks-runner/task-graph-utils.js +17 -12
  90. package/dist/src/tasks-runner/task-orchestrator.d.ts +37 -8
  91. package/dist/src/tasks-runner/task-orchestrator.js +460 -76
  92. package/dist/src/tasks-runner/tasks-schedule.d.ts +18 -1
  93. package/dist/src/tasks-runner/tasks-schedule.js +64 -6
  94. package/dist/src/tasks-runner/utils.d.ts +20 -5
  95. package/dist/src/tasks-runner/utils.js +70 -35
  96. package/dist/src/ultracache/overrides.js +4 -0
  97. package/dist/src/utils/child-process.js +1 -0
  98. package/dist/src/utils/git-utils.d.ts +6 -1
  99. package/dist/src/utils/git-utils.js +26 -5
  100. package/dist/src/utils/owned-private-dir.d.ts +1 -17
  101. package/dist/src/utils/owned-private-dir.js +10 -62
  102. package/dist/src/utils/perf-hooks.d.ts +1 -0
  103. package/dist/src/utils/plugins/core-plugins.js +1 -1
  104. package/dist/src/utils/spawn-without-terminal.d.ts +27 -0
  105. package/dist/src/utils/spawn-without-terminal.js +110 -0
  106. package/dist/src/utils/string-similarity.d.ts +38 -0
  107. package/dist/src/utils/string-similarity.js +108 -0
  108. package/migrations.json +7 -0
  109. package/package.json +11 -20
  110. package/readme-template.md +3 -3
  111. package/schemas/nx-schema.json +70 -0
  112. package/schemas/project-schema.json +67 -0
  113. package/dist/src/tasks-runner/legacy-depends-on-warning.d.ts +0 -18
  114. package/dist/src/tasks-runner/legacy-depends-on-warning.js +0 -109
@@ -124,6 +124,7 @@ export const installNxConsole = __napiModule.exports.installNxConsole
124
124
  export const IS_WASM = __napiModule.exports.IS_WASM
125
125
  export const isAiAgent = __napiModule.exports.isAiAgent
126
126
  export const logDebug = __napiModule.exports.logDebug
127
+ export const openUrl = __napiModule.exports.openUrl
127
128
  export const remove = __napiModule.exports.remove
128
129
  export const testOnlyTransferFileMap = __napiModule.exports.testOnlyTransferFileMap
129
130
  export const transferProjectGraph = __napiModule.exports.transferProjectGraph
@@ -154,6 +154,7 @@ module.exports.installNxConsole = __napiModule.exports.installNxConsole
154
154
  module.exports.IS_WASM = __napiModule.exports.IS_WASM
155
155
  module.exports.isAiAgent = __napiModule.exports.isAiAgent
156
156
  module.exports.logDebug = __napiModule.exports.logDebug
157
+ module.exports.openUrl = __napiModule.exports.openUrl
157
158
  module.exports.remove = __napiModule.exports.remove
158
159
  module.exports.testOnlyTransferFileMap = __napiModule.exports.testOnlyTransferFileMap
159
160
  module.exports.transferProjectGraph = __napiModule.exports.transferProjectGraph
Binary file
@@ -1,7 +1,12 @@
1
+ import { HttpClient } from '../utils/http-client';
1
2
  import type { CloudTaskRunnerOptions } from './nx-cloud-tasks-runner-shell';
2
3
  import type { ReadUltracacheConfigurationsOptions, ReadUltracacheConfigurationsResult } from '../ultracache/fetch';
3
4
  import { TasksRunner } from '../tasks-runner/tasks-runner';
4
5
  import { RemoteCacheV2 } from '../tasks-runner/default-tasks-runner';
6
+ interface CloudBundleInstall {
7
+ version: string;
8
+ fullPath: string;
9
+ }
5
10
  export declare class NxCloudEnterpriseOutdatedError extends Error {
6
11
  constructor(url: string);
7
12
  }
@@ -24,3 +29,5 @@ export declare function verifyOrUpdateNxCloudClient(options?: {
24
29
  version: string;
25
30
  } | null>;
26
31
  export declare function getBundleInstallDefaultLocation(): string;
32
+ export declare function downloadAndExtractClientBundle(httpClient: HttpClient, version: string, url: string): Promise<CloudBundleInstall>;
33
+ export {};
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.NxCloudClientUnavailableError = exports.NxCloudEnterpriseOutdatedError = void 0;
4
4
  exports.verifyOrUpdateNxCloudClient = verifyOrUpdateNxCloudClient;
5
5
  exports.getBundleInstallDefaultLocation = getBundleInstallDefaultLocation;
6
+ exports.downloadAndExtractClientBundle = downloadAndExtractClientBundle;
6
7
  const tslib_1 = require("tslib");
7
8
  const fs_1 = require("fs");
8
9
  const zlib_1 = require("zlib");
@@ -16,6 +17,7 @@ const tar = tslib_1.__importStar(require("tar-stream"));
16
17
  const cache_directory_1 = require("../utils/cache-directory");
17
18
  const is_ci_1 = require("../utils/is-ci");
18
19
  const crypto_1 = require("crypto");
20
+ const native_1 = require("../native");
19
21
  const workspace_root_1 = require("../utils/workspace-root");
20
22
  class NxCloudEnterpriseOutdatedError extends Error {
21
23
  constructor(url) {
@@ -74,13 +76,13 @@ async function verifyOrUpdateNxCloudClient(options) {
74
76
  if (version === 'NX_ENTERPRISE_OUTDATED_IMAGE') {
75
77
  throw new NxCloudEnterpriseOutdatedError(apiUrl);
76
78
  }
77
- const fullPath = await downloadAndExtractClientBundle(httpClient, runnerBundleInstallDirectory, version, url);
78
- (0, debug_logger_1.debugLog)('Done: ', fullPath);
79
- const nxCloudClient = require(fullPath);
79
+ const installedBundle = await downloadAndExtractClientBundle(httpClient, version, url);
80
+ (0, debug_logger_1.debugLog)('Done: ', installedBundle.fullPath);
81
+ const nxCloudClient = require(installedBundle.fullPath);
80
82
  if (nxCloudClient.commands === undefined) {
81
83
  throw new NxCloudEnterpriseOutdatedError(apiUrl);
82
84
  }
83
- return { version, nxCloudClient };
85
+ return { version: installedBundle.version, nxCloudClient };
84
86
  }
85
87
  if (currentBundle === null) {
86
88
  throw new NxCloudClientUnavailableError();
@@ -114,6 +116,25 @@ function getBundleInstallDefaultLocation() {
114
116
  return (0, path_1.join)((0, is_ci_1.isCI)() ? (0, cache_directory_1.cacheDirectoryForWorkspace)(workspace_root_1.workspaceRoot) : cache_directory_1.cacheDir, 'cloud');
115
117
  }
116
118
  const runnerBundleInstallDirectory = getBundleInstallDefaultLocation();
119
+ // Control files live outside the install directory. Released nx versions load
120
+ // any directory there as a bundle, so they would `require` a state directory
121
+ // placed inside it.
122
+ const stateDirectory = (0, fs_1.existsSync)((0, path_1.join)(workspace_root_1.workspaceRoot, 'nx.json'))
123
+ ? (0, path_1.join)(cache_directory_1.workspaceDataDirectory, 'nx-cloud')
124
+ : `${runnerBundleInstallDirectory}-state`;
125
+ function ensureStateDirectory() {
126
+ (0, fs_1.mkdirSync)(stateDirectory, { recursive: true });
127
+ }
128
+ const downloadLockFilePath = (0, path_1.join)(stateDirectory, 'download.lock');
129
+ // The record lives beside the lockfile rather than inside it. Windows
130
+ // byte-range locks are mandatory and handle-scoped, so writing to a file this
131
+ // process holds an exclusive lock on fails with ERROR_LOCK_VIOLATION. It also
132
+ // matches the convention elsewhere in nx: project-graph.lock and run.json.lock
133
+ // are never written to.
134
+ const downloadRecordFilePath = (0, path_1.join)(stateDirectory, 'download.record');
135
+ // A version names a directory that is created and later deleted, so a value
136
+ // from the server must not be able to escape the install directory.
137
+ const VALID_BUNDLE_VERSION = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
117
138
  function getLatestInstalledRunnerBundle() {
118
139
  if (!(0, fs_1.existsSync)(runnerBundleInstallDirectory)) {
119
140
  (0, fs_1.mkdirSync)(runnerBundleInstallDirectory, { recursive: true });
@@ -121,7 +142,9 @@ function getLatestInstalledRunnerBundle() {
121
142
  try {
122
143
  const installedBundles = (0, fs_1.readdirSync)(runnerBundleInstallDirectory)
123
144
  .filter((potentialDirectory) => {
124
- return (0, fs_1.statSync)((0, path_1.join)(runnerBundleInstallDirectory, potentialDirectory)).isDirectory();
145
+ // '.tmp-*' directories are in-progress or crashed downloads
146
+ return (!potentialDirectory.startsWith('.') &&
147
+ (0, fs_1.statSync)((0, path_1.join)(runnerBundleInstallDirectory, potentialDirectory)).isDirectory());
125
148
  })
126
149
  .map((fileOrDirectory) => ({
127
150
  version: fileOrDirectory,
@@ -131,7 +154,13 @@ function getLatestInstalledRunnerBundle() {
131
154
  // No installed bundles
132
155
  return null;
133
156
  }
134
- return installedBundles[0];
157
+ // A contended install can leave several bundles on disk. The record names
158
+ // the directory an install last completed in, which is the one to run.
159
+ // Without a record the directory name is all there is to go on and is read
160
+ // as the version, which is right for a bundle an older nx installed and
161
+ // otherwise yields a version the server rejects, costing one more
162
+ // download.
163
+ return recordedBundle() ?? installedBundles[0];
135
164
  }
136
165
  catch (e) {
137
166
  console.log('Could not read runner bundle path:', e.message);
@@ -170,7 +199,7 @@ async function verifyCurrentBundle(httpClient, currentBundle) {
170
199
  });
171
200
  }
172
201
  function getLatestBundleVerificationTimestamp() {
173
- const lockfilePath = (0, path_1.join)(runnerBundleInstallDirectory, 'verify.lock');
202
+ const lockfilePath = (0, path_1.join)(stateDirectory, 'verify.lock');
174
203
  if ((0, fs_1.existsSync)(lockfilePath)) {
175
204
  const timestampAsString = (0, fs_1.readFileSync)(lockfilePath, 'utf-8');
176
205
  let timestampAsNumber;
@@ -185,7 +214,8 @@ function getLatestBundleVerificationTimestamp() {
185
214
  return null;
186
215
  }
187
216
  function writeBundleVerificationLock() {
188
- const lockfilePath = (0, path_1.join)(runnerBundleInstallDirectory, 'verify.lock');
217
+ const lockfilePath = (0, path_1.join)(stateDirectory, 'verify.lock');
218
+ ensureStateDirectory();
189
219
  (0, fs_1.writeFileSync)(lockfilePath, new Date().getTime().toString(), 'utf-8');
190
220
  }
191
221
  function getBundleContentHash(bundle) {
@@ -211,7 +241,118 @@ function hashDirectory(dir) {
211
241
  const combinedHashes = hashes.sort().join('');
212
242
  return (0, crypto_1.createHash)('sha256').update(combinedHashes).digest('hex');
213
243
  }
214
- async function downloadAndExtractClientBundle(httpClient, runnerBundleInstallDirectory, version, url) {
244
+ async function downloadAndExtractClientBundle(httpClient, version, url) {
245
+ // Parallel nx processes race to install bundles, possibly at different
246
+ // versions. The first to take the lock downloads; the rest wait and adopt
247
+ // its bundle when an install completed during the wait at the version the
248
+ // server asked them for. Otherwise they download their own as a contended
249
+ // install, which publishes its own directory and skips cleanup, so the
250
+ // holder's bundle stays on disk for the process running from it - at the
251
+ // same version as well as a different one. The flock is released by the
252
+ // kernel if the holder dies, so no stale-lock cleanup is needed. Under WASM
253
+ // the lock is unavailable and downloads run unserialized.
254
+ if (!VALID_BUNDLE_VERSION.test(version)) {
255
+ throw new Error(`Invalid Nx Cloud client bundle version: ${version}`);
256
+ }
257
+ const recordBeforeContending = readDownloadRecord();
258
+ const lock = !native_1.IS_WASM ? new native_1.FileLock(downloadLockFilePath) : null;
259
+ let locked = lock?.locked;
260
+ let contended = false;
261
+ while (locked) {
262
+ (0, debug_logger_1.debugLog)('Another process is downloading the client bundle, waiting for it to complete');
263
+ await lock.wait();
264
+ // The holder may be running from a bundle whose record predates this call,
265
+ // so the wait alone makes the install contended.
266
+ contended = true;
267
+ const installedBundle = bundleInstalledSince(recordBeforeContending);
268
+ if (installedBundle) {
269
+ if (installedBundle.version === version) {
270
+ (0, debug_logger_1.debugLog)('Using client bundle downloaded by another process: ', installedBundle.version);
271
+ return installedBundle;
272
+ }
273
+ (0, debug_logger_1.debugLog)('Another process installed a different bundle: ', installedBundle.version);
274
+ }
275
+ // The other process failed or installed an older version, so this process
276
+ // still needs to download.
277
+ locked = lock.check();
278
+ }
279
+ lock?.lock();
280
+ try {
281
+ // A process that acquired the lock between the check above and lock() may
282
+ // have completed an install already.
283
+ const installedBundle = bundleInstalledSince(recordBeforeContending);
284
+ if (installedBundle) {
285
+ if (installedBundle.version === version) {
286
+ (0, debug_logger_1.debugLog)('Using client bundle downloaded by another process: ', installedBundle.version);
287
+ return installedBundle;
288
+ }
289
+ // A different version means that process is running from a bundle this
290
+ // one must not delete.
291
+ contended = true;
292
+ (0, debug_logger_1.debugLog)('Another process installed a different bundle: ', installedBundle.version);
293
+ }
294
+ const fullPath = await downloadAndExtractBundle(httpClient, version, url, contended);
295
+ return { version, fullPath };
296
+ }
297
+ finally {
298
+ lock?.unlock();
299
+ }
300
+ }
301
+ // Records "<serverVersion> <directoryName>", written only once an install has
302
+ // COMPLETED. The directory is recorded separately because it cannot be derived
303
+ // from the version - every install publishes its own (see
304
+ // downloadAndExtractBundle), so several directories can carry the same
305
+ // version. That also makes every record distinct, so a waiter can tell "an
306
+ // install finished while I waited" from "this record is left over from a past
307
+ // run" by comparing the record it read before contending. Timestamps cannot
308
+ // answer that: filesystem mtime granularity is coarser than the race window.
309
+ function readDownloadRecord() {
310
+ try {
311
+ return (0, fs_1.readFileSync)(downloadRecordFilePath, 'utf-8').trim();
312
+ }
313
+ catch {
314
+ return '';
315
+ }
316
+ }
317
+ function writeDownloadRecord(version, directoryName) {
318
+ ensureStateDirectory();
319
+ (0, fs_1.writeFileSync)(downloadRecordFilePath, `${version} ${directoryName}`, 'utf-8');
320
+ }
321
+ /**
322
+ * The bundle the record names, if it is still on disk. A record whose second
323
+ * field does not name a live directory resolves to null and leaves selection
324
+ * on its fallback, which is what a one-field record or anything else
325
+ * unexpected in the file has to do.
326
+ */
327
+ function recordedBundle() {
328
+ const [version, directoryName] = readDownloadRecord().split(' ');
329
+ if (!version ||
330
+ !directoryName ||
331
+ !VALID_BUNDLE_VERSION.test(version) ||
332
+ !VALID_BUNDLE_VERSION.test(directoryName)) {
333
+ return null;
334
+ }
335
+ const fullPath = (0, path_1.join)(runnerBundleInstallDirectory, directoryName);
336
+ return (0, fs_1.existsSync)(fullPath) ? { version, fullPath } : null;
337
+ }
338
+ /**
339
+ * The bundle an install completed during this call, or null. Requires the
340
+ * record to have changed since `recordBefore` was read: a directory existing
341
+ * proves only that some earlier run left one there, and an install
342
+ * interrupted on a released nx leaves exactly that.
343
+ */
344
+ function bundleInstalledSince(recordBefore) {
345
+ return readDownloadRecord() === recordBefore ? null : recordedBundle();
346
+ }
347
+ async function downloadAndExtractBundle(httpClient, version, url, contended) {
348
+ // Every install publishes its own directory. Two processes can be asked for
349
+ // the same version while one of them is already running from a completed
350
+ // install of it, so replacing <installDir>/<version> in place would delete
351
+ // that bundle out from under it - and a holder loading its client in the gap
352
+ // fails with MODULE_NOT_FOUND, which drops the task runner to running
353
+ // without Nx Cloud. A directory nothing else has seen cannot be in use.
354
+ const bundleDirectoryName = `${version}-${(0, crypto_1.randomUUID)()}`;
355
+ const bundleExtractLocation = (0, path_1.join)(runnerBundleInstallDirectory, bundleDirectoryName);
215
356
  let resp;
216
357
  try {
217
358
  resp = await httpClient.get(url, {
@@ -222,53 +363,99 @@ async function downloadAndExtractClientBundle(httpClient, runnerBundleInstallDir
222
363
  console.error('Error while updating Nx Cloud client bundle');
223
364
  throw e;
224
365
  }
225
- const bundleExtractLocation = (0, path_1.join)(runnerBundleInstallDirectory, version);
226
- if (!(0, fs_1.existsSync)(bundleExtractLocation)) {
227
- (0, fs_1.mkdirSync)(bundleExtractLocation);
228
- }
229
- return new Promise((res, rej) => {
230
- const extract = tar.extract();
231
- extract.on('entry', function (headers, stream, next) {
232
- if (headers.type === 'directory') {
233
- const directoryPath = (0, path_1.join)(bundleExtractLocation, headers.name);
234
- if (!(0, fs_1.existsSync)(directoryPath)) {
235
- (0, fs_1.mkdirSync)(directoryPath, { recursive: true });
236
- }
237
- next();
238
- stream.resume();
239
- }
240
- else if (headers.type === 'file') {
241
- const outputFilePath = (0, path_1.join)(bundleExtractLocation, headers.name);
242
- const writeStream = (0, fs_1.createWriteStream)(outputFilePath);
243
- // Surface disk errors through the pipeline instead of crashing
244
- writeStream.on('error', (e) => extract.destroy(e));
245
- stream.pipe(writeStream);
246
- // Continue the tar stream after the write stream closes
247
- writeStream.on('close', () => {
366
+ // Extract into a temp directory and rename into place afterwards, so a
367
+ // failed or interrupted download never leaves a partial bundle at the
368
+ // path other processes require it from.
369
+ const tempExtractLocation = (0, path_1.join)(runnerBundleInstallDirectory, `.tmp-${version}-${process.pid}`);
370
+ (0, fs_1.mkdirSync)(tempExtractLocation, { recursive: true });
371
+ try {
372
+ await new Promise((res, rej) => {
373
+ const extract = tar.extract();
374
+ extract.on('entry', function (headers, stream, next) {
375
+ if (headers.type === 'directory') {
376
+ const directoryPath = (0, path_1.join)(tempExtractLocation, headers.name);
377
+ if (!(0, fs_1.existsSync)(directoryPath)) {
378
+ (0, fs_1.mkdirSync)(directoryPath, { recursive: true });
379
+ }
248
380
  next();
249
- });
250
- stream.resume();
251
- }
252
- });
253
- extract.on('finish', function () {
254
- removeOldClientBundles(version);
255
- writeBundleVerificationLock();
256
- res(bundleExtractLocation);
257
- });
258
- // pipeline propagates download/gunzip errors that .pipe() would leave
259
- // uncaught
260
- (0, stream_1.pipeline)(resp.data, (0, zlib_1.createGunzip)(), extract, (e) => {
261
- if (e)
262
- rej(e);
381
+ stream.resume();
382
+ }
383
+ else if (headers.type === 'file') {
384
+ const outputFilePath = (0, path_1.join)(tempExtractLocation, headers.name);
385
+ const writeStream = (0, fs_1.createWriteStream)(outputFilePath);
386
+ // Surface disk errors through the pipeline instead of crashing
387
+ writeStream.on('error', (e) => extract.destroy(e));
388
+ stream.pipe(writeStream);
389
+ // Continue the tar stream after the write stream closes
390
+ writeStream.on('close', () => {
391
+ next();
392
+ });
393
+ stream.resume();
394
+ }
395
+ else {
396
+ // Skipping the entry would publish an incomplete bundle.
397
+ extract.destroy(new Error(`Unsupported ${headers.type} entry in Nx Cloud client bundle: ${headers.name}`));
398
+ }
399
+ });
400
+ extract.on('finish', function () {
401
+ res();
402
+ });
403
+ // pipeline propagates download/gunzip errors that .pipe() would leave
404
+ // uncaught
405
+ (0, stream_1.pipeline)(resp.data, (0, zlib_1.createGunzip)(), extract, (e) => {
406
+ if (e)
407
+ rej(e);
408
+ });
263
409
  });
264
- });
410
+ (0, fs_1.renameSync)(tempExtractLocation, bundleExtractLocation);
411
+ // Recorded only now: the record is the signal that a bundle at this
412
+ // version was installed by this process, which is what lets a waiter
413
+ // adopt it instead of downloading again, and it carries the directory
414
+ // that selection has to load from.
415
+ writeDownloadRecord(version, bundleDirectoryName);
416
+ }
417
+ catch (e) {
418
+ (0, fs_1.rmSync)(tempExtractLocation, { recursive: true, force: true });
419
+ throw e;
420
+ }
421
+ // On a contended install another process may be running from any bundle on
422
+ // disk, now including one at this same version, so nothing here can be shown
423
+ // to be unused and collection waits for a later uncontended install. Repeat
424
+ // contention accumulates directories until one of those runs. Bounding that
425
+ // means knowing which bundles are still loaded, which nothing on disk
426
+ // records today.
427
+ if (!contended) {
428
+ removeOldClientBundles(bundleDirectoryName);
429
+ }
430
+ writeBundleVerificationLock();
431
+ return bundleExtractLocation;
265
432
  }
266
- function removeOldClientBundles(currentInstallVersion) {
433
+ function removeOldClientBundles(currentInstallDirectoryName) {
267
434
  const filesAndFolders = (0, fs_1.readdirSync)(runnerBundleInstallDirectory);
268
435
  for (let fileOrFolder of filesAndFolders) {
436
+ // A '.tmp-*' left by a crashed extract is reclaimed here, which is safe
437
+ // only because this runs after our own was renamed away and the download
438
+ // lock means no other process is extracting. Under WASM there is no lock,
439
+ // so a concurrent extract may still own that directory: leaving it costs
440
+ // disk, deleting it would fail that process's install.
441
+ if (fileOrFolder === currentInstallDirectoryName ||
442
+ (native_1.IS_WASM && fileOrFolder.startsWith('.tmp-'))) {
443
+ continue;
444
+ }
269
445
  const fileOrFolderPath = (0, path_1.join)(runnerBundleInstallDirectory, fileOrFolder);
270
- if (fileOrFolder !== currentInstallVersion) {
271
- (0, fs_1.rmSync)(fileOrFolderPath, { recursive: true });
446
+ // Another process's cleanup can remove an entry between the readdir above
447
+ // and these calls, so both tolerate it already being gone.
448
+ let isBundle;
449
+ try {
450
+ // Only directories are bundles. Files here, like the verify.lock a
451
+ // released nx writes, are left alone.
452
+ isBundle = (0, fs_1.statSync)(fileOrFolderPath).isDirectory();
453
+ }
454
+ catch {
455
+ continue;
456
+ }
457
+ if (isBundle) {
458
+ (0, fs_1.rmSync)(fileOrFolderPath, { recursive: true, force: true });
272
459
  }
273
460
  }
274
461
  }
@@ -148,6 +148,7 @@ async function selectAffectedTasks(projectGraph, nxJson, planningContext, reques
148
148
  const dependencies = dependencyChanges(projectGraph, touchedFiles, nxJson, packageJson);
149
149
  const namedProjects = new Set(dependencies.projects);
150
150
  const projects = (0, project_graph_1.readProjectsConfigurationFromProjectGraph)(projectGraph).projects;
151
+ // TODO(v25): remove custom hasher support.
151
152
  const customHashed = new Set(taskIds.filter((id) => hasCustomHasher(taskGraph.tasks[id], projects)));
152
153
  const options = {
153
154
  projectGlobPatterns: await (0, affected_projects_1.getProjectGlobPatterns)(nxJson),
@@ -351,6 +351,13 @@ class IsolatedPlugin {
351
351
  this.shutdown();
352
352
  }
353
353
  async setWorkerEnv(env) {
354
+ // A restart may have spawned under the previous client env but not yet
355
+ // finished loading. Wait for that worker so it receives the update too.
356
+ // Do not start a stopped worker solely to forward env: its next spawn
357
+ // will inherit the daemon's already-updated environment.
358
+ if (!this._alive && this._connectPromise) {
359
+ await this._connectPromise;
360
+ }
354
361
  if (!this._alive) {
355
362
  return;
356
363
  }
@@ -1,5 +1,5 @@
1
1
  import { ProjectGraph, ProjectGraphProjectNode } from '../config/project-graph';
2
- import { Task, TaskGraph } from '../config/task-graph';
2
+ import { Task, TaskGraph, TaskGraphEdge } from '../config/task-graph';
3
3
  import { TargetDependencies } from '../config/nx-json';
4
4
  /**
5
5
  * For each task, the overrides every `dependsOn` edge into it would give it,
@@ -20,10 +20,10 @@ export declare class ProcessTasks {
20
20
  [id: string]: Task;
21
21
  };
22
22
  readonly dependencies: {
23
- [k: string]: string[];
23
+ [k: string]: TaskGraphEdge[];
24
24
  };
25
25
  readonly continuousDependencies: {
26
- [k: string]: string[];
26
+ [k: string]: TaskGraphEdge[];
27
27
  };
28
28
  readonly dependencyOverrides: DependencyOverrides;
29
29
  private readonly allTargetNames;
@@ -33,6 +33,7 @@ export declare class ProcessTasks {
33
33
  private processTasksForMultipleProjects;
34
34
  private processTasksForSingleProject;
35
35
  private processTasksForDependencies;
36
+ private addContinuousEdge;
36
37
  private recordEdge;
37
38
  private createDummyTask;
38
39
  createTask(id: string, project: ProjectGraphProjectNode, target: string, resolvedConfiguration: string | undefined, overrides: Record<string, unknown>): Task;
@@ -54,11 +55,11 @@ export declare function filterTaskGraphToSelection(projectGraph: ProjectGraph, t
54
55
  * It will manipulate the dependencies object in place
55
56
  */
56
57
  export declare function filterDummyTasks(dependencies: {
57
- [k: string]: string[];
58
+ [k: string]: TaskGraphEdge[];
58
59
  }): void;
59
60
  /**
60
61
  * this function is used to get the non dummy dependencies of a task recursively
61
62
  */
62
- export declare function getNonDummyDeps(currentTask: string, dependencies: {
63
- [k: string]: string[];
64
- }, cycles?: Set<string>, seen?: Set<string>): string[];
63
+ export declare function getNonDummyDeps(currentEdge: TaskGraphEdge, dependencies: {
64
+ [k: string]: TaskGraphEdge[];
65
+ }, cycles?: Set<string>, seen?: Set<string>): TaskGraphEdge[];
@@ -59,28 +59,14 @@ class ProcessTasks {
59
59
  }
60
60
  }
61
61
  for (let d of Object.keys(this.dependencies)) {
62
- this.dependencies[d] = this.dependencies[d].filter((dd) => !!initialTasks[dd]);
62
+ this.dependencies[d] = this.dependencies[d].filter((dd) => !!initialTasks[dd.id]);
63
63
  }
64
64
  for (let d of Object.keys(this.continuousDependencies)) {
65
- this.continuousDependencies[d] = this.continuousDependencies[d].filter((dd) => !!initialTasks[dd]);
65
+ this.continuousDependencies[d] = this.continuousDependencies[d].filter((dd) => !!initialTasks[dd.id]);
66
66
  }
67
67
  }
68
68
  filterDummyTasks(this.dependencies);
69
- for (const taskId of Object.keys(this.dependencies)) {
70
- if (this.dependencies[taskId].length > 0) {
71
- this.dependencies[taskId] = [
72
- ...new Set(this.dependencies[taskId].filter((d) => d !== taskId)).values(),
73
- ];
74
- }
75
- }
76
69
  filterDummyTasks(this.continuousDependencies);
77
- for (const taskId of Object.keys(this.continuousDependencies)) {
78
- if (this.continuousDependencies[taskId].length > 0) {
79
- this.continuousDependencies[taskId] = [
80
- ...new Set(this.continuousDependencies[taskId].filter((d) => d !== taskId)).values(),
81
- ];
82
- }
83
- }
84
70
  return Object.keys(this.tasks).filter((d) => this.dependencies[d].length === 0 &&
85
71
  this.continuousDependencies[d].length === 0);
86
72
  }
@@ -132,10 +118,10 @@ class ProcessTasks {
132
118
  if (task.id !== selfTaskId) {
133
119
  this.recordEdge(task, selfTaskId, taskOverrides);
134
120
  if (this.tasks[selfTaskId].continuous) {
135
- this.continuousDependencies[task.id].push(selfTaskId);
121
+ this.addContinuousEdge(task, selfTaskId, dependencyConfig);
136
122
  }
137
123
  else {
138
- this.dependencies[task.id].push(selfTaskId);
124
+ this.dependencies[task.id].push({ id: selfTaskId });
139
125
  }
140
126
  }
141
127
  }
@@ -156,10 +142,10 @@ class ProcessTasks {
156
142
  if (task.id !== depTargetId) {
157
143
  this.recordEdge(task, depTargetId, taskOverrides);
158
144
  if (depTargetConfiguration.continuous) {
159
- this.continuousDependencies[task.id].push(depTargetId);
145
+ this.addContinuousEdge(task, depTargetId, dependencyConfig);
160
146
  }
161
147
  else {
162
- this.dependencies[task.id].push(depTargetId);
148
+ this.dependencies[task.id].push({ id: depTargetId });
163
149
  }
164
150
  }
165
151
  if (!this.tasks[depTargetId]) {
@@ -177,8 +163,8 @@ class ProcessTasks {
177
163
  '__' +
178
164
  dependencyConfig.target +
179
165
  DUMMY_TASK_TARGET, undefined);
180
- this.dependencies[task.id].push(dummyId);
181
- this.continuousDependencies[task.id].push(dummyId);
166
+ this.dependencies[task.id].push({ id: dummyId });
167
+ this.continuousDependencies[task.id].push({ id: dummyId });
182
168
  this.dependencies[dummyId] ??= [];
183
169
  this.continuousDependencies[dummyId] ??= [];
184
170
  const noopTask = this.createDummyTask(dummyId, task);
@@ -186,6 +172,9 @@ class ProcessTasks {
186
172
  }
187
173
  }
188
174
  }
175
+ addContinuousEdge(task, dependencyId, dependencyConfig) {
176
+ this.continuousDependencies[task.id].push(continuousEdge(dependencyId, dependencyConfig));
177
+ }
189
178
  recordEdge(task, dependencyId, overrides) {
190
179
  if (!this.recordDependencyOverrides) {
191
180
  return;
@@ -298,6 +287,37 @@ function interpolateOverrides(args, projectName, project) {
298
287
  });
299
288
  return interpolatedArgs;
300
289
  }
290
+ function continuousEdge(id, dependencyConfig) {
291
+ const edge = { id };
292
+ if (dependencyConfig.waitFor === 'ready') {
293
+ edge.waitFor = 'ready';
294
+ }
295
+ if (dependencyConfig.inputs === false) {
296
+ edge.inputs = false;
297
+ }
298
+ return edge;
299
+ }
300
+ // A task reached by several edges waits for ready when any of them does, and
301
+ // keeps its inputs unless all of them opt out
302
+ function mergeEdges(edges) {
303
+ const byId = new Map();
304
+ for (const edge of edges) {
305
+ const seen = byId.get(edge.id);
306
+ if (!seen) {
307
+ byId.set(edge.id, edge);
308
+ continue;
309
+ }
310
+ const merged = { id: edge.id };
311
+ if (seen.waitFor === 'ready' || edge.waitFor === 'ready') {
312
+ merged.waitFor = 'ready';
313
+ }
314
+ if (seen.inputs === false && edge.inputs === false) {
315
+ merged.inputs = false;
316
+ }
317
+ byId.set(edge.id, merged);
318
+ }
319
+ return [...byId.values()];
320
+ }
301
321
  /**
302
322
  * This function is used to filter out the dummy tasks from the dependencies
303
323
  * It will manipulate the dependencies object in place
@@ -310,7 +330,7 @@ function filterDummyTasks(dependencies) {
310
330
  for (const dep of deps) {
311
331
  normalizedDeps.push(...getNonDummyDeps(dep, dependencies, cycles, new Set([key])));
312
332
  }
313
- dependencies[key] = normalizedDeps;
333
+ dependencies[key] = mergeEdges(normalizedDeps);
314
334
  }
315
335
  }
316
336
  for (const key of Object.keys(dependencies)) {
@@ -322,12 +342,13 @@ function filterDummyTasks(dependencies) {
322
342
  /**
323
343
  * this function is used to get the non dummy dependencies of a task recursively
324
344
  */
325
- function getNonDummyDeps(currentTask, dependencies, cycles, seen = new Set()) {
345
+ function getNonDummyDeps(currentEdge, dependencies, cycles, seen = new Set()) {
346
+ const currentTask = currentEdge.id;
326
347
  if (seen.has(currentTask)) {
327
348
  return [];
328
349
  }
329
- seen.add(currentTask);
330
350
  if (currentTask.endsWith(DUMMY_TASK_TARGET)) {
351
+ seen.add(currentTask);
331
352
  if (cycles?.has(currentTask)) {
332
353
  return [];
333
354
  }
@@ -339,7 +360,7 @@ function getNonDummyDeps(currentTask, dependencies, cycles, seen = new Set()) {
339
360
  return deps.flatMap((dep) => getNonDummyDeps(dep, dependencies, cycles, seen));
340
361
  }
341
362
  else {
342
- return [currentTask];
363
+ return [currentEdge];
343
364
  }
344
365
  }
345
366
  function createTaskOverrides(dependencyConfig, cliOverrides, sourceTask, projectGraph) {
@@ -13,6 +13,7 @@ const shared_running_task_1 = require("./running-tasks/shared-running-task");
13
13
  const task_results_life_cycle_1 = require("./life-cycles/task-results-life-cycle");
14
14
  async function createOrchestrator(tasks, projectGraph, fullTaskGraph, nxJson, lifeCycle, ultracacheConfigurations) {
15
15
  (0, dotenv_1.loadRootEnvFiles)();
16
+ fullTaskGraph = withEdgeObjects(fullTaskGraph);
16
17
  const invokeRunnerTerminalLifecycle = new invoke_runner_terminal_output_life_cycle_1.InvokeRunnerTerminalOutputLifeCycle(tasks);
17
18
  const taskResultsLifecycle = new task_results_life_cycle_1.TaskResultsLifeCycle();
18
19
  const compositedLifeCycle = new life_cycle_1.CompositeLifeCycle([
@@ -52,6 +53,18 @@ async function createOrchestrator(tasks, projectGraph, fullTaskGraph, nxJson, li
52
53
  orchestrator.processAllScheduledTasks();
53
54
  return orchestrator;
54
55
  }
56
+ // A client bundle built before edges became objects hands plain task ids
57
+ function withEdgeObjects(taskGraph) {
58
+ const normalize = (edges) => Object.fromEntries(Object.entries(edges).map(([taskId, taskEdges]) => [
59
+ taskId,
60
+ taskEdges.map((edge) => typeof edge === 'string' ? { id: edge } : edge),
61
+ ]));
62
+ return {
63
+ ...taskGraph,
64
+ dependencies: normalize(taskGraph.dependencies),
65
+ continuousDependencies: normalize(taskGraph.continuousDependencies),
66
+ };
67
+ }
55
68
  // Nothing awaits the dispose chains below, so an unhandled rejection would take
56
69
  // down a long-lived agent process.
57
70
  function logDisposeFailure(e) {