@atlaspack/core 2.13.2-dev.3682 → 2.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/CHANGELOG.md +61 -0
  2. package/package.json +20 -20
  3. package/src/Atlaspack.js +3 -2
  4. package/src/atlaspack-v3/AtlaspackV3.js +24 -8
  5. package/src/loadAtlaspackPlugin.js +26 -71
  6. package/src/projectPath.js +14 -1
  7. package/src/requests/WriteBundlesRequest.js +31 -22
  8. package/LICENSE +0 -201
  9. package/lib/AssetGraph.js +0 -521
  10. package/lib/Atlaspack.js +0 -676
  11. package/lib/AtlaspackConfig.js +0 -298
  12. package/lib/AtlaspackConfig.schema.js +0 -103
  13. package/lib/BundleGraph.js +0 -1527
  14. package/lib/CommittedAsset.js +0 -155
  15. package/lib/Dependency.js +0 -136
  16. package/lib/Environment.js +0 -144
  17. package/lib/IdentifierRegistry.js +0 -36
  18. package/lib/InternalConfig.js +0 -56
  19. package/lib/PackagerRunner.js +0 -525
  20. package/lib/ReporterRunner.js +0 -151
  21. package/lib/RequestTracker.js +0 -1178
  22. package/lib/SymbolPropagation.js +0 -618
  23. package/lib/TargetDescriptor.schema.js +0 -118
  24. package/lib/Transformation.js +0 -522
  25. package/lib/UncommittedAsset.js +0 -348
  26. package/lib/Validation.js +0 -203
  27. package/lib/applyRuntimes.js +0 -355
  28. package/lib/assetUtils.js +0 -205
  29. package/lib/atlaspack-v3/AtlaspackV3.js +0 -57
  30. package/lib/atlaspack-v3/NapiWorkerPool.js +0 -71
  31. package/lib/atlaspack-v3/fs.js +0 -39
  32. package/lib/atlaspack-v3/index.js +0 -26
  33. package/lib/atlaspack-v3/jsCallable.js +0 -20
  34. package/lib/atlaspack-v3/worker/compat/asset-symbols.js +0 -197
  35. package/lib/atlaspack-v3/worker/compat/bitflags.js +0 -84
  36. package/lib/atlaspack-v3/worker/compat/dependency.js +0 -44
  37. package/lib/atlaspack-v3/worker/compat/environment.js +0 -57
  38. package/lib/atlaspack-v3/worker/compat/index.js +0 -104
  39. package/lib/atlaspack-v3/worker/compat/mutable-asset.js +0 -164
  40. package/lib/atlaspack-v3/worker/compat/plugin-config.js +0 -78
  41. package/lib/atlaspack-v3/worker/compat/plugin-logger.js +0 -29
  42. package/lib/atlaspack-v3/worker/compat/plugin-options.js +0 -113
  43. package/lib/atlaspack-v3/worker/compat/plugin-tracer.js +0 -12
  44. package/lib/atlaspack-v3/worker/compat/target.js +0 -17
  45. package/lib/atlaspack-v3/worker/index.js +0 -3
  46. package/lib/atlaspack-v3/worker/worker.js +0 -280
  47. package/lib/constants.js +0 -21
  48. package/lib/dumpGraphToGraphViz.js +0 -206
  49. package/lib/index.js +0 -70
  50. package/lib/loadAtlaspackPlugin.js +0 -136
  51. package/lib/loadDotEnv.js +0 -54
  52. package/lib/projectPath.js +0 -94
  53. package/lib/public/Asset.js +0 -259
  54. package/lib/public/Bundle.js +0 -236
  55. package/lib/public/BundleGraph.js +0 -279
  56. package/lib/public/BundleGroup.js +0 -50
  57. package/lib/public/Config.js +0 -202
  58. package/lib/public/Dependency.js +0 -131
  59. package/lib/public/Environment.js +0 -247
  60. package/lib/public/MutableBundleGraph.js +0 -204
  61. package/lib/public/PluginOptions.js +0 -71
  62. package/lib/public/Symbols.js +0 -247
  63. package/lib/public/Target.js +0 -64
  64. package/lib/registerCoreWithSerializer.js +0 -51
  65. package/lib/requests/AssetGraphRequest.js +0 -432
  66. package/lib/requests/AssetGraphRequestRust.js +0 -220
  67. package/lib/requests/AssetRequest.js +0 -132
  68. package/lib/requests/AtlaspackBuildRequest.js +0 -79
  69. package/lib/requests/AtlaspackConfigRequest.js +0 -479
  70. package/lib/requests/BundleGraphRequest.js +0 -485
  71. package/lib/requests/ConfigRequest.js +0 -203
  72. package/lib/requests/DevDepRequest.js +0 -193
  73. package/lib/requests/EntryRequest.js +0 -295
  74. package/lib/requests/PackageRequest.js +0 -88
  75. package/lib/requests/PathRequest.js +0 -357
  76. package/lib/requests/TargetRequest.js +0 -1179
  77. package/lib/requests/ValidationRequest.js +0 -66
  78. package/lib/requests/WriteBundleRequest.js +0 -252
  79. package/lib/requests/WriteBundlesRequest.js +0 -153
  80. package/lib/requests/asset-graph-diff.js +0 -128
  81. package/lib/requests/asset-graph-dot.js +0 -131
  82. package/lib/resolveOptions.js +0 -265
  83. package/lib/serializerCore.browser.js +0 -29
  84. package/lib/summarizeRequest.js +0 -55
  85. package/lib/types.js +0 -35
  86. package/lib/utils.js +0 -160
  87. package/lib/worker.js +0 -184
@@ -1,1178 +0,0 @@
1
- "use strict";
2
-
3
- Object.defineProperty(exports, "__esModule", {
4
- value: true
5
- });
6
- exports.RequestGraph = void 0;
7
- exports.cleanUpOrphans = cleanUpOrphans;
8
- exports.default = void 0;
9
- exports.getWatcherOptions = getWatcherOptions;
10
- exports.readAndDeserializeRequestGraph = readAndDeserializeRequestGraph;
11
- exports.requestTypes = exports.requestGraphEdgeTypes = void 0;
12
- function _assert() {
13
- const data = _interopRequireWildcard(require("assert"));
14
- _assert = function () {
15
- return data;
16
- };
17
- return data;
18
- }
19
- function _path2() {
20
- const data = _interopRequireDefault(require("path"));
21
- _path2 = function () {
22
- return data;
23
- };
24
- return data;
25
- }
26
- function _buildCache() {
27
- const data = require("@atlaspack/build-cache");
28
- _buildCache = function () {
29
- return data;
30
- };
31
- return data;
32
- }
33
- function _featureFlags() {
34
- const data = require("@atlaspack/feature-flags");
35
- _featureFlags = function () {
36
- return data;
37
- };
38
- return data;
39
- }
40
- function _graph() {
41
- const data = require("@atlaspack/graph");
42
- _graph = function () {
43
- return data;
44
- };
45
- return data;
46
- }
47
- function _logger() {
48
- const data = _interopRequireDefault(require("@atlaspack/logger"));
49
- _logger = function () {
50
- return data;
51
- };
52
- return data;
53
- }
54
- function _rust() {
55
- const data = require("@atlaspack/rust");
56
- _rust = function () {
57
- return data;
58
- };
59
- return data;
60
- }
61
- function _utils() {
62
- const data = require("@atlaspack/utils");
63
- _utils = function () {
64
- return data;
65
- };
66
- return data;
67
- }
68
- function _nullthrows() {
69
- const data = _interopRequireDefault(require("nullthrows"));
70
- _nullthrows = function () {
71
- return data;
72
- };
73
- return data;
74
- }
75
- var _constants = require("./constants");
76
- var _projectPath = require("./projectPath");
77
- var _ReporterRunner = require("./ReporterRunner");
78
- var _ConfigRequest = require("./requests/ConfigRequest");
79
- var _utils2 = require("./utils");
80
- function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
81
- function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
82
- function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
83
- const requestGraphEdgeTypes = exports.requestGraphEdgeTypes = {
84
- subrequest: 2,
85
- invalidated_by_update: 3,
86
- invalidated_by_delete: 4,
87
- invalidated_by_create: 5,
88
- invalidated_by_create_above: 6,
89
- dirname: 7
90
- };
91
- class FSBailoutError extends Error {
92
- name = 'FSBailoutError';
93
- }
94
- const FILE = 0;
95
- const REQUEST = 1;
96
- const FILE_NAME = 2;
97
- const ENV = 3;
98
- const OPTION = 4;
99
- const GLOB = 5;
100
- const CONFIG_KEY = 6;
101
- const requestTypes = exports.requestTypes = {
102
- atlaspack_build_request: 1,
103
- bundle_graph_request: 2,
104
- asset_graph_request: 3,
105
- entry_request: 4,
106
- target_request: 5,
107
- atlaspack_config_request: 6,
108
- path_request: 7,
109
- dev_dep_request: 8,
110
- asset_request: 9,
111
- config_request: 10,
112
- write_bundles_request: 11,
113
- package_request: 12,
114
- write_bundle_request: 13,
115
- validation_request: 14
116
- };
117
- const nodeFromFilePath = filePath => ({
118
- id: (0, _projectPath.fromProjectPathRelative)(filePath),
119
- type: FILE
120
- });
121
- const nodeFromGlob = glob => ({
122
- id: (0, _projectPath.fromProjectPathRelative)(glob),
123
- type: GLOB,
124
- value: glob
125
- });
126
- const nodeFromFileName = fileName => ({
127
- id: 'file_name:' + fileName,
128
- type: FILE_NAME
129
- });
130
- const nodeFromRequest = request => ({
131
- id: request.id,
132
- type: REQUEST,
133
- requestType: request.requestType,
134
- invalidateReason: _constants.INITIAL_BUILD
135
- });
136
- const nodeFromEnv = (env, value) => ({
137
- id: 'env:' + env,
138
- type: ENV,
139
- value
140
- });
141
- const nodeFromOption = (option, value) => ({
142
- id: 'option:' + option,
143
- type: OPTION,
144
- hash: (0, _utils2.hashFromOption)(value)
145
- });
146
- const nodeFromConfigKey = (fileName, configKey, contentHash) => ({
147
- id: `config_key:${(0, _projectPath.fromProjectPathRelative)(fileName)}:${configKey}`,
148
- type: CONFIG_KEY,
149
- configKey,
150
- contentHash
151
- });
152
- const keyFromEnvContentKey = contentKey => contentKey.slice('env:'.length);
153
- const keyFromOptionContentKey = contentKey => contentKey.slice('option:'.length);
154
-
155
- // This constant is chosen by local profiling the time to serialise n nodes and tuning until an average time of ~50 ms per blob.
156
- // The goal is to free up the event loop periodically to allow interruption by the user.
157
- const NODES_PER_BLOB = 2 ** 14;
158
- class RequestGraph extends _graph().ContentGraph {
159
- invalidNodeIds = new Set();
160
- incompleteNodeIds = new Set();
161
- incompleteNodePromises = new Map();
162
- globNodeIds = new Set();
163
- envNodeIds = new Set();
164
- optionNodeIds = new Set();
165
- // Unpredictable nodes are requests that cannot be predicted whether they should rerun based on
166
- // filesystem changes alone. They should rerun on each startup of Atlaspack.
167
- unpredicatableNodeIds = new Set();
168
- invalidateOnBuildNodeIds = new Set();
169
- cachedRequestChunks = new Set();
170
- configKeyNodes = new Map();
171
- nodesPerBlob = NODES_PER_BLOB;
172
-
173
- // $FlowFixMe[prop-missing]
174
- static deserialize(opts) {
175
- // $FlowFixMe[prop-missing]
176
- let deserialized = new RequestGraph(opts);
177
- deserialized.invalidNodeIds = opts.invalidNodeIds;
178
- deserialized.incompleteNodeIds = opts.incompleteNodeIds;
179
- deserialized.globNodeIds = opts.globNodeIds;
180
- deserialized.envNodeIds = opts.envNodeIds;
181
- deserialized.optionNodeIds = opts.optionNodeIds;
182
- deserialized.unpredicatableNodeIds = opts.unpredicatableNodeIds;
183
- deserialized.invalidateOnBuildNodeIds = opts.invalidateOnBuildNodeIds;
184
- deserialized.cachedRequestChunks = opts.cachedRequestChunks;
185
- deserialized.configKeyNodes = opts.configKeyNodes;
186
- return deserialized;
187
- }
188
-
189
- // $FlowFixMe[prop-missing]
190
- serialize() {
191
- return {
192
- ...super.serialize(),
193
- invalidNodeIds: this.invalidNodeIds,
194
- incompleteNodeIds: this.incompleteNodeIds,
195
- globNodeIds: this.globNodeIds,
196
- envNodeIds: this.envNodeIds,
197
- optionNodeIds: this.optionNodeIds,
198
- unpredicatableNodeIds: this.unpredicatableNodeIds,
199
- invalidateOnBuildNodeIds: this.invalidateOnBuildNodeIds,
200
- cachedRequestChunks: this.cachedRequestChunks,
201
- configKeyNodes: this.configKeyNodes
202
- };
203
- }
204
-
205
- // addNode for RequestGraph should not override the value if added multiple times
206
- addNode(node) {
207
- let nodeId = this._contentKeyToNodeId.get(node.id);
208
- if (nodeId != null) {
209
- return nodeId;
210
- }
211
- nodeId = super.addNodeByContentKey(node.id, node);
212
- if (node.type === GLOB) {
213
- this.globNodeIds.add(nodeId);
214
- } else if (node.type === ENV) {
215
- this.envNodeIds.add(nodeId);
216
- } else if (node.type === OPTION) {
217
- this.optionNodeIds.add(nodeId);
218
- }
219
- this.removeCachedRequestChunkForNode(nodeId);
220
- return nodeId;
221
- }
222
- removeNode(nodeId, removeOrphans = true) {
223
- this.invalidNodeIds.delete(nodeId);
224
- this.incompleteNodeIds.delete(nodeId);
225
- this.incompleteNodePromises.delete(nodeId);
226
- this.unpredicatableNodeIds.delete(nodeId);
227
- this.invalidateOnBuildNodeIds.delete(nodeId);
228
- let node = (0, _nullthrows().default)(this.getNode(nodeId));
229
- if (node.type === GLOB) {
230
- this.globNodeIds.delete(nodeId);
231
- } else if (node.type === ENV) {
232
- this.envNodeIds.delete(nodeId);
233
- } else if (node.type === OPTION) {
234
- this.optionNodeIds.delete(nodeId);
235
- } else if (node.type === CONFIG_KEY) {
236
- for (let configKeyNodes of this.configKeyNodes.values()) {
237
- configKeyNodes.delete(nodeId);
238
- }
239
- }
240
- return super.removeNode(nodeId, removeOrphans);
241
- }
242
- getRequestNode(nodeId) {
243
- let node = (0, _nullthrows().default)(this.getNode(nodeId));
244
- if (node.type === REQUEST) {
245
- return node;
246
- }
247
- throw new (_assert().AssertionError)({
248
- message: `Expected a request node: ${node.type} (${typeof node.type}) does not equal ${REQUEST} (${typeof REQUEST}).`,
249
- expected: REQUEST,
250
- actual: node.type
251
- });
252
- }
253
- replaceSubrequests(requestNodeId, subrequestContentKeys) {
254
- let subrequestNodeIds = [];
255
- for (let key of subrequestContentKeys) {
256
- if (this.hasContentKey(key)) {
257
- subrequestNodeIds.push(this.getNodeIdByContentKey(key));
258
- }
259
- }
260
- this.replaceNodeIdsConnectedTo(requestNodeId, subrequestNodeIds, null, requestGraphEdgeTypes.subrequest);
261
- }
262
- invalidateNode(nodeId, reason) {
263
- let node = (0, _nullthrows().default)(this.getNode(nodeId));
264
- (0, _assert().default)(node.type === REQUEST);
265
- node.invalidateReason |= reason;
266
- this.invalidNodeIds.add(nodeId);
267
- let parentNodes = this.getNodeIdsConnectedTo(nodeId, requestGraphEdgeTypes.subrequest);
268
- for (let parentNode of parentNodes) {
269
- this.invalidateNode(parentNode, reason);
270
- }
271
-
272
- // If the node is invalidated, the cached request chunk on disk needs to be re-written
273
- this.removeCachedRequestChunkForNode(nodeId);
274
- }
275
- invalidateUnpredictableNodes() {
276
- for (let nodeId of this.unpredicatableNodeIds) {
277
- let node = (0, _nullthrows().default)(this.getNode(nodeId));
278
- (0, _assert().default)(node.type !== FILE && node.type !== GLOB);
279
- this.invalidateNode(nodeId, _constants.STARTUP);
280
- }
281
- }
282
- invalidateOnBuildNodes() {
283
- for (let nodeId of this.invalidateOnBuildNodeIds) {
284
- let node = (0, _nullthrows().default)(this.getNode(nodeId));
285
- (0, _assert().default)(node.type !== FILE && node.type !== GLOB);
286
- this.invalidateNode(nodeId, _constants.STARTUP);
287
- }
288
- }
289
- invalidateEnvNodes(env) {
290
- for (let nodeId of this.envNodeIds) {
291
- let node = (0, _nullthrows().default)(this.getNode(nodeId));
292
- (0, _assert().default)(node.type === ENV);
293
- if (env[keyFromEnvContentKey(node.id)] !== node.value) {
294
- let parentNodes = this.getNodeIdsConnectedTo(nodeId, requestGraphEdgeTypes.invalidated_by_update);
295
- for (let parentNode of parentNodes) {
296
- this.invalidateNode(parentNode, _constants.ENV_CHANGE);
297
- }
298
- }
299
- }
300
- }
301
- invalidateOptionNodes(options) {
302
- for (let nodeId of this.optionNodeIds) {
303
- let node = (0, _nullthrows().default)(this.getNode(nodeId));
304
- (0, _assert().default)(node.type === OPTION);
305
- if ((0, _utils2.hashFromOption)(options[keyFromOptionContentKey(node.id)]) !== node.hash) {
306
- let parentNodes = this.getNodeIdsConnectedTo(nodeId, requestGraphEdgeTypes.invalidated_by_update);
307
- for (let parentNode of parentNodes) {
308
- this.invalidateNode(parentNode, _constants.OPTION_CHANGE);
309
- }
310
- }
311
- }
312
- }
313
- invalidateOnConfigKeyChange(requestNodeId, filePath, configKey, contentHash) {
314
- let configKeyNodeId = this.addNode(nodeFromConfigKey(filePath, configKey, contentHash));
315
- let nodes = this.configKeyNodes.get(filePath);
316
- if (!nodes) {
317
- nodes = new Set();
318
- this.configKeyNodes.set(filePath, nodes);
319
- }
320
- nodes.add(configKeyNodeId);
321
- if (!this.hasEdge(requestNodeId, configKeyNodeId, requestGraphEdgeTypes.invalidated_by_update)) {
322
- this.addEdge(requestNodeId, configKeyNodeId,
323
- // Store as an update edge, but file deletes are handled too
324
- requestGraphEdgeTypes.invalidated_by_update);
325
- }
326
- }
327
- invalidateOnFileUpdate(requestNodeId, filePath) {
328
- let fileNodeId = this.addNode(nodeFromFilePath(filePath));
329
- if (!this.hasEdge(requestNodeId, fileNodeId, requestGraphEdgeTypes.invalidated_by_update)) {
330
- this.addEdge(requestNodeId, fileNodeId, requestGraphEdgeTypes.invalidated_by_update);
331
- }
332
- }
333
- invalidateOnFileDelete(requestNodeId, filePath) {
334
- let fileNodeId = this.addNode(nodeFromFilePath(filePath));
335
- if (!this.hasEdge(requestNodeId, fileNodeId, requestGraphEdgeTypes.invalidated_by_delete)) {
336
- this.addEdge(requestNodeId, fileNodeId, requestGraphEdgeTypes.invalidated_by_delete);
337
- }
338
- }
339
- invalidateOnFileCreate(requestNodeId, input) {
340
- let node;
341
- if (input.glob != null) {
342
- node = nodeFromGlob(input.glob);
343
- } else if (input.fileName != null && input.aboveFilePath != null) {
344
- let aboveFilePath = input.aboveFilePath;
345
-
346
- // Create nodes and edges for each part of the filename pattern.
347
- // For example, 'node_modules/foo' would create two nodes and one edge.
348
- // This creates a sort of trie structure within the graph that can be
349
- // quickly matched by following the edges. This is also memory efficient
350
- // since common sub-paths (e.g. 'node_modules') are deduplicated.
351
- let parts = input.fileName.split('/').reverse();
352
- let lastNodeId;
353
- for (let part of parts) {
354
- let fileNameNode = nodeFromFileName(part);
355
- let fileNameNodeId = this.addNode(fileNameNode);
356
- if (lastNodeId != null && !this.hasEdge(lastNodeId, fileNameNodeId, requestGraphEdgeTypes.dirname)) {
357
- this.addEdge(lastNodeId, fileNameNodeId, requestGraphEdgeTypes.dirname);
358
- }
359
- lastNodeId = fileNameNodeId;
360
- }
361
-
362
- // The `aboveFilePath` condition asserts that requests are only invalidated
363
- // if the file being created is "above" it in the filesystem (e.g. the file
364
- // is created in a parent directory). There is likely to already be a node
365
- // for this file in the graph (e.g. the source file) that we can reuse for this.
366
- node = nodeFromFilePath(aboveFilePath);
367
- let nodeId = this.addNode(node);
368
-
369
- // Now create an edge from the `aboveFilePath` node to the first file_name node
370
- // in the chain created above, and an edge from the last node in the chain back to
371
- // the `aboveFilePath` node. When matching, we will start from the first node in
372
- // the chain, and continue following it to parent directories until there is an
373
- // edge pointing an `aboveFilePath` node that also points to the start of the chain.
374
- // This indicates a complete match, and any requests attached to the `aboveFilePath`
375
- // node will be invalidated.
376
- let firstId = 'file_name:' + parts[0];
377
- let firstNodeId = this.getNodeIdByContentKey(firstId);
378
- if (!this.hasEdge(nodeId, firstNodeId, requestGraphEdgeTypes.invalidated_by_create_above)) {
379
- this.addEdge(nodeId, firstNodeId, requestGraphEdgeTypes.invalidated_by_create_above);
380
- }
381
- (0, _assert().default)(lastNodeId != null);
382
- if (!this.hasEdge(lastNodeId, nodeId, requestGraphEdgeTypes.invalidated_by_create_above)) {
383
- this.addEdge(lastNodeId, nodeId, requestGraphEdgeTypes.invalidated_by_create_above);
384
- }
385
- } else if (input.filePath != null) {
386
- node = nodeFromFilePath(input.filePath);
387
- } else {
388
- throw new Error('Invalid invalidation');
389
- }
390
- let nodeId = this.addNode(node);
391
- if (!this.hasEdge(requestNodeId, nodeId, requestGraphEdgeTypes.invalidated_by_create)) {
392
- this.addEdge(requestNodeId, nodeId, requestGraphEdgeTypes.invalidated_by_create);
393
- }
394
- }
395
- invalidateOnStartup(requestNodeId) {
396
- this.getRequestNode(requestNodeId);
397
- this.unpredicatableNodeIds.add(requestNodeId);
398
- }
399
- invalidateOnBuild(requestNodeId) {
400
- this.getRequestNode(requestNodeId);
401
- this.invalidateOnBuildNodeIds.add(requestNodeId);
402
- }
403
- invalidateOnEnvChange(requestNodeId, env, value) {
404
- let envNode = nodeFromEnv(env, value);
405
- let envNodeId = this.addNode(envNode);
406
- if (!this.hasEdge(requestNodeId, envNodeId, requestGraphEdgeTypes.invalidated_by_update)) {
407
- this.addEdge(requestNodeId, envNodeId, requestGraphEdgeTypes.invalidated_by_update);
408
- }
409
- }
410
- invalidateOnOptionChange(requestNodeId, option, value) {
411
- let optionNode = nodeFromOption(option, value);
412
- let optionNodeId = this.addNode(optionNode);
413
- if (!this.hasEdge(requestNodeId, optionNodeId, requestGraphEdgeTypes.invalidated_by_update)) {
414
- this.addEdge(requestNodeId, optionNodeId, requestGraphEdgeTypes.invalidated_by_update);
415
- }
416
- }
417
- clearInvalidations(nodeId) {
418
- this.unpredicatableNodeIds.delete(nodeId);
419
- this.invalidateOnBuildNodeIds.delete(nodeId);
420
- this.replaceNodeIdsConnectedTo(nodeId, [], null, requestGraphEdgeTypes.invalidated_by_update);
421
- this.replaceNodeIdsConnectedTo(nodeId, [], null, requestGraphEdgeTypes.invalidated_by_delete);
422
- this.replaceNodeIdsConnectedTo(nodeId, [], null, requestGraphEdgeTypes.invalidated_by_create);
423
- }
424
- getInvalidations(requestNodeId) {
425
- if (!this.hasNode(requestNodeId)) {
426
- return [];
427
- }
428
-
429
- // For now just handling updates. Could add creates/deletes later if needed.
430
- let invalidations = this.getNodeIdsConnectedFrom(requestNodeId, requestGraphEdgeTypes.invalidated_by_update);
431
- return invalidations.map(nodeId => {
432
- let node = (0, _nullthrows().default)(this.getNode(nodeId));
433
- switch (node.type) {
434
- case FILE:
435
- return {
436
- type: 'file',
437
- filePath: (0, _projectPath.toProjectPathUnsafe)(node.id)
438
- };
439
- case ENV:
440
- return {
441
- type: 'env',
442
- key: keyFromEnvContentKey(node.id)
443
- };
444
- case OPTION:
445
- return {
446
- type: 'option',
447
- key: keyFromOptionContentKey(node.id)
448
- };
449
- }
450
- }).filter(Boolean);
451
- }
452
- getSubRequests(requestNodeId) {
453
- if (!this.hasNode(requestNodeId)) {
454
- return [];
455
- }
456
- let subRequests = this.getNodeIdsConnectedFrom(requestNodeId, requestGraphEdgeTypes.subrequest);
457
- return subRequests.map(nodeId => {
458
- let node = (0, _nullthrows().default)(this.getNode(nodeId));
459
- (0, _assert().default)(node.type === REQUEST);
460
- return node;
461
- });
462
- }
463
- getInvalidSubRequests(requestNodeId) {
464
- if (!this.hasNode(requestNodeId)) {
465
- return [];
466
- }
467
- let subRequests = this.getNodeIdsConnectedFrom(requestNodeId, requestGraphEdgeTypes.subrequest);
468
- return subRequests.filter(id => this.invalidNodeIds.has(id)).map(nodeId => {
469
- let node = (0, _nullthrows().default)(this.getNode(nodeId));
470
- (0, _assert().default)(node.type === REQUEST);
471
- return node;
472
- });
473
- }
474
- invalidateFileNameNode(node, filePath, matchNodes, invalidateNode) {
475
- // If there is an edge between this file_name node and one of the original file nodes pointed to
476
- // by the original file_name node, and the matched node is inside the current directory, invalidate
477
- // all connected requests pointed to by the file node.
478
-
479
- let nodeId = this.getNodeIdByContentKey(node.id);
480
- let dirname = _path2().default.dirname((0, _projectPath.fromProjectPathRelative)(filePath));
481
- if ((0, _featureFlags().getFeatureFlag)('fixQuadraticCacheInvalidation')) {
482
- while (dirname !== '/') {
483
- if (!this.hasContentKey(dirname)) break;
484
- const matchNodeId = this.getNodeIdByContentKey(dirname);
485
- if (!this.hasEdge(nodeId, matchNodeId, requestGraphEdgeTypes.invalidated_by_create_above)) break;
486
- const connectedNodes = this.getNodeIdsConnectedTo(matchNodeId, requestGraphEdgeTypes.invalidated_by_create);
487
- for (let connectedNode of connectedNodes) {
488
- invalidateNode(connectedNode, _constants.FILE_CREATE);
489
- }
490
- dirname = _path2().default.dirname(dirname);
491
- }
492
- } else {
493
- for (let matchNode of matchNodes) {
494
- let matchNodeId = this.getNodeIdByContentKey(matchNode.id);
495
- if (this.hasEdge(nodeId, matchNodeId, requestGraphEdgeTypes.invalidated_by_create_above) && (0, _utils().isDirectoryInside)((0, _projectPath.fromProjectPathRelative)((0, _projectPath.toProjectPathUnsafe)(matchNode.id)), dirname)) {
496
- let connectedNodes = this.getNodeIdsConnectedTo(matchNodeId, requestGraphEdgeTypes.invalidated_by_create);
497
- for (let connectedNode of connectedNodes) {
498
- this.invalidateNode(connectedNode, _constants.FILE_CREATE);
499
- }
500
- }
501
- }
502
- }
503
-
504
- // Find the `file_name` node for the parent directory and
505
- // recursively invalidate connected requests as described above.
506
- let basename = _path2().default.basename(dirname);
507
- let contentKey = 'file_name:' + basename;
508
- if (this.hasContentKey(contentKey)) {
509
- if (this.hasEdge(nodeId, this.getNodeIdByContentKey(contentKey), requestGraphEdgeTypes.dirname)) {
510
- let parent = (0, _nullthrows().default)(this.getNodeByContentKey(contentKey));
511
- (0, _assert().default)(parent.type === FILE_NAME);
512
- this.invalidateFileNameNode(parent, (0, _projectPath.toProjectPathUnsafe)(dirname), matchNodes, invalidateNode);
513
- }
514
- }
515
- }
516
- async respondToFSEvents(events, options, threshold,
517
- /**
518
- * True if this is the start-up (loading phase) invalidation.
519
- */
520
- isInitialBuild = false) {
521
- let didInvalidate = false;
522
- let count = 0;
523
- let predictedTime = 0;
524
- let startTime = Date.now();
525
- const enableOptimization = (0, _featureFlags().getFeatureFlag)('fixQuadraticCacheInvalidation');
526
- const removeOrphans = !enableOptimization;
527
- const invalidatedNodes = new Set();
528
- const invalidateNode = (nodeId, reason) => {
529
- if (enableOptimization && invalidatedNodes.has(nodeId)) {
530
- return;
531
- }
532
- invalidatedNodes.add(nodeId);
533
- this.invalidateNode(nodeId, reason);
534
- };
535
- const aboveCache = new Map();
536
- const getAbove = fileNameNodeId => {
537
- const cachedResult = aboveCache.get(fileNameNodeId);
538
- if (enableOptimization && cachedResult) {
539
- return cachedResult;
540
- }
541
- let above = [];
542
- const children = this.getNodeIdsConnectedTo(fileNameNodeId, requestGraphEdgeTypes.invalidated_by_create_above);
543
- for (const nodeId of children) {
544
- let node = (0, _nullthrows().default)(this.getNode(nodeId));
545
- if (node.type === FILE) {
546
- above.push(node);
547
- }
548
- }
549
- aboveCache.set(fileNameNodeId, above);
550
- return above;
551
- };
552
- for (let {
553
- path: _path,
554
- type
555
- } of events) {
556
- if (!enableOptimization && process.env.ATLASPACK_DISABLE_CACHE_TIMEOUT !== 'true' && ++count === 256) {
557
- let duration = Date.now() - startTime;
558
- predictedTime = duration * (events.length >> 8);
559
- if (predictedTime > threshold) {
560
- _logger().default.warn({
561
- origin: '@atlaspack/core',
562
- message: 'Building with clean cache. Cache invalidation took too long.',
563
- meta: {
564
- trackableEvent: 'cache_invalidation_timeout',
565
- watcherEventCount: events.length,
566
- predictedTime
567
- }
568
- });
569
- throw new FSBailoutError('Responding to file system events exceeded threshold, start with empty cache.');
570
- }
571
- }
572
- let _filePath = (0, _projectPath.toProjectPath)(options.projectRoot, _path);
573
- let filePath = (0, _projectPath.fromProjectPathRelative)(_filePath);
574
- let hasFileRequest = this.hasContentKey(filePath);
575
-
576
- // If we see a 'create' event for the project root itself,
577
- // this means the project root was moved and we need to
578
- // re-run all requests.
579
- if (type === 'create' && filePath === '') {
580
- _logger().default.verbose({
581
- origin: '@atlaspack/core',
582
- message: 'Watcher reported project root create event. Invalidate all nodes.',
583
- meta: {
584
- trackableEvent: 'project_root_create'
585
- }
586
- });
587
- for (let [id, node] of this.nodes.entries()) {
588
- if ((node === null || node === void 0 ? void 0 : node.type) === REQUEST) {
589
- this.invalidNodeIds.add(id);
590
- }
591
- }
592
- return true;
593
- }
594
-
595
- // sometimes mac os reports update events as create events.
596
- // if it was a create event, but the file already exists in the graph,
597
- // then also invalidate nodes connected by invalidated_by_update edges.
598
- if (hasFileRequest && (type === 'create' || type === 'update')) {
599
- let nodeId = this.getNodeIdByContentKey(filePath);
600
- let nodes = this.getNodeIdsConnectedTo(nodeId, requestGraphEdgeTypes.invalidated_by_update);
601
- for (let connectedNode of nodes) {
602
- didInvalidate = true;
603
- invalidateNode(connectedNode, _constants.FILE_UPDATE);
604
- }
605
- if (type === 'create') {
606
- let nodes = this.getNodeIdsConnectedTo(nodeId, requestGraphEdgeTypes.invalidated_by_create);
607
- for (let connectedNode of nodes) {
608
- didInvalidate = true;
609
- invalidateNode(connectedNode, _constants.FILE_CREATE);
610
- }
611
- }
612
- } else if (type === 'create') {
613
- let basename = _path2().default.basename(filePath);
614
- let fileNameNode = this.getNodeByContentKey('file_name:' + basename);
615
- if (fileNameNode != null && fileNameNode.type === FILE_NAME) {
616
- let fileNameNodeId = this.getNodeIdByContentKey('file_name:' + basename);
617
-
618
- // Find potential file nodes to be invalidated if this file name pattern matches
619
- let above = getAbove(fileNameNodeId);
620
- if (above.length > 0) {
621
- didInvalidate = true;
622
- this.invalidateFileNameNode(fileNameNode, _filePath, above, invalidateNode);
623
- }
624
- }
625
- for (let globeNodeId of this.globNodeIds) {
626
- let globNode = this.getNode(globeNodeId);
627
- (0, _assert().default)(globNode && globNode.type === GLOB);
628
- if ((0, _utils().isGlobMatch)(filePath, (0, _projectPath.fromProjectPathRelative)(globNode.value))) {
629
- let connectedNodes = this.getNodeIdsConnectedTo(globeNodeId, requestGraphEdgeTypes.invalidated_by_create);
630
- for (let connectedNode of connectedNodes) {
631
- didInvalidate = true;
632
- invalidateNode(connectedNode, _constants.FILE_CREATE);
633
- }
634
- }
635
- }
636
- } else if (hasFileRequest && type === 'delete') {
637
- let nodeId = this.getNodeIdByContentKey(filePath);
638
- for (let connectedNode of this.getNodeIdsConnectedTo(nodeId, requestGraphEdgeTypes.invalidated_by_delete)) {
639
- didInvalidate = true;
640
- invalidateNode(connectedNode, _constants.FILE_DELETE);
641
- }
642
-
643
- // Delete the file node since it doesn't exist anymore.
644
- // This ensures that files that don't exist aren't sent
645
- // to requests as invalidations for future requests.
646
- this.removeNode(nodeId, removeOrphans);
647
- }
648
- let configKeyNodes = this.configKeyNodes.get(_filePath);
649
- if (configKeyNodes && (type === 'delete' || type === 'update')) {
650
- for (let nodeId of configKeyNodes) {
651
- let isInvalid = type === 'delete';
652
- if (type === 'update') {
653
- let node = this.getNode(nodeId);
654
- (0, _assert().default)(node && node.type === CONFIG_KEY);
655
- let contentHash = await (0, _ConfigRequest.getConfigKeyContentHash)(_filePath, node.configKey, options);
656
- isInvalid = node.contentHash !== contentHash;
657
- }
658
- if (isInvalid) {
659
- for (let connectedNode of this.getNodeIdsConnectedTo(nodeId, requestGraphEdgeTypes.invalidated_by_update)) {
660
- invalidateNode(connectedNode, type === 'delete' ? _constants.FILE_DELETE : _constants.FILE_UPDATE);
661
- }
662
- didInvalidate = true;
663
- this.removeNode(nodeId, removeOrphans);
664
- }
665
- }
666
- }
667
- }
668
- if ((0, _featureFlags().getFeatureFlag)('fixQuadraticCacheInvalidation')) {
669
- cleanUpOrphans(this);
670
- }
671
- let duration = Date.now() - startTime;
672
- _logger().default.verbose({
673
- origin: '@atlaspack/core',
674
- message: `RequestGraph.respondToFSEvents duration: ${duration}`,
675
- meta: {
676
- trackableEvent: 'fsevent_response_time',
677
- duration,
678
- predictedTime,
679
- isInitialBuild,
680
- numberOfEvents: events.length,
681
- numberOfInvalidatedNodes: invalidatedNodes.size
682
- }
683
- });
684
- return didInvalidate && this.invalidNodeIds.size > 0;
685
- }
686
- hasCachedRequestChunk(index) {
687
- return this.cachedRequestChunks.has(index);
688
- }
689
- setCachedRequestChunk(index) {
690
- this.cachedRequestChunks.add(index);
691
- }
692
- removeCachedRequestChunkForNode(nodeId) {
693
- this.cachedRequestChunks.delete(Math.floor(nodeId / this.nodesPerBlob));
694
- }
695
- }
696
- exports.RequestGraph = RequestGraph;
697
- class RequestTracker {
698
- stats = new Map();
699
- constructor({
700
- graph,
701
- farm,
702
- options,
703
- rustAtlaspack
704
- }) {
705
- this.graph = graph || new RequestGraph();
706
- this.farm = farm;
707
- this.options = options;
708
- this.rustAtlaspack = rustAtlaspack;
709
- }
710
-
711
- // TODO: refactor (abortcontroller should be created by RequestTracker)
712
- setSignal(signal) {
713
- this.signal = signal;
714
- }
715
- startRequest(request) {
716
- let didPreviouslyExist = this.graph.hasContentKey(request.id);
717
- let requestNodeId;
718
- if (didPreviouslyExist) {
719
- requestNodeId = this.graph.getNodeIdByContentKey(request.id);
720
- // Clear existing invalidations for the request so that the new
721
- // invalidations created during the request replace the existing ones.
722
- this.graph.clearInvalidations(requestNodeId);
723
- } else {
724
- requestNodeId = this.graph.addNode(nodeFromRequest(request));
725
- }
726
- this.graph.incompleteNodeIds.add(requestNodeId);
727
- this.graph.invalidNodeIds.delete(requestNodeId);
728
- let {
729
- promise,
730
- deferred
731
- } = (0, _utils().makeDeferredWithPromise)();
732
- this.graph.incompleteNodePromises.set(requestNodeId, promise);
733
- return {
734
- requestNodeId,
735
- deferred
736
- };
737
- }
738
-
739
- // If a cache key is provided, the result will be removed from the node and stored in a separate cache entry
740
- storeResult(nodeId, result, cacheKey) {
741
- let node = this.graph.getNode(nodeId);
742
- if (node && node.type === REQUEST) {
743
- node.result = result;
744
- node.resultCacheKey = cacheKey;
745
- }
746
- }
747
- hasValidResult(nodeId) {
748
- return this.graph.hasNode(nodeId) && !this.graph.invalidNodeIds.has(nodeId) && !this.graph.incompleteNodeIds.has(nodeId);
749
- }
750
- async getRequestResult(contentKey, ifMatch) {
751
- let node = (0, _nullthrows().default)(this.graph.getNodeByContentKey(contentKey));
752
- (0, _assert().default)(node.type === REQUEST);
753
- if (ifMatch != null && node.resultCacheKey !== ifMatch) {
754
- return null;
755
- }
756
- if (node.result != undefined) {
757
- // $FlowFixMe
758
- let result = node.result;
759
- return result;
760
- } else if (node.resultCacheKey != null && ifMatch == null) {
761
- let key = node.resultCacheKey;
762
- (0, _assert().default)(this.options.cache.hasLargeBlob(key));
763
- let cachedResult = (0, _buildCache().deserialize)(await this.options.cache.getLargeBlob(key));
764
- node.result = cachedResult;
765
- return cachedResult;
766
- }
767
- }
768
- completeRequest(nodeId) {
769
- this.graph.invalidNodeIds.delete(nodeId);
770
- this.graph.incompleteNodeIds.delete(nodeId);
771
- this.graph.incompleteNodePromises.delete(nodeId);
772
- let node = this.graph.getNode(nodeId);
773
- if (node && node.type === REQUEST) {
774
- node.invalidateReason = _constants.VALID;
775
- }
776
- this.graph.removeCachedRequestChunkForNode(nodeId);
777
- }
778
- rejectRequest(nodeId) {
779
- this.graph.incompleteNodeIds.delete(nodeId);
780
- this.graph.incompleteNodePromises.delete(nodeId);
781
- let node = this.graph.getNode(nodeId);
782
- if ((node === null || node === void 0 ? void 0 : node.type) === REQUEST) {
783
- this.graph.invalidateNode(nodeId, _constants.ERROR);
784
- }
785
- }
786
- respondToFSEvents(events, threshold) {
787
- return this.graph.respondToFSEvents(events, this.options, threshold);
788
- }
789
- hasInvalidRequests() {
790
- return this.graph.invalidNodeIds.size > 0;
791
- }
792
- getInvalidRequests() {
793
- let invalidRequests = [];
794
- for (let id of this.graph.invalidNodeIds) {
795
- let node = (0, _nullthrows().default)(this.graph.getNode(id));
796
- (0, _assert().default)(node.type === REQUEST);
797
- invalidRequests.push(node);
798
- }
799
- return invalidRequests;
800
- }
801
- replaceSubrequests(requestNodeId, subrequestContextKeys) {
802
- this.graph.replaceSubrequests(requestNodeId, subrequestContextKeys);
803
- }
804
- async runRequest(request, opts) {
805
- let hasKey = this.graph.hasContentKey(request.id);
806
- let requestId = hasKey ? this.graph.getNodeIdByContentKey(request.id) : undefined;
807
- let hasValidResult = requestId != null && this.hasValidResult(requestId);
808
- if (!(opts !== null && opts !== void 0 && opts.force) && hasValidResult) {
809
- // $FlowFixMe[incompatible-type]
810
- return this.getRequestResult(request.id);
811
- }
812
- if (requestId != null) {
813
- let incompletePromise = this.graph.incompleteNodePromises.get(requestId);
814
- if (incompletePromise != null) {
815
- // There is a another instance of this request already running, wait for its completion and reuse its result
816
- try {
817
- if (await incompletePromise) {
818
- // $FlowFixMe[incompatible-type]
819
- return this.getRequestResult(request.id);
820
- }
821
- } catch (e) {
822
- // Rerun this request
823
- }
824
- }
825
- }
826
- let previousInvalidations = requestId != null ? this.graph.getInvalidations(requestId) : [];
827
- let {
828
- requestNodeId,
829
- deferred
830
- } = this.startRequest({
831
- id: request.id,
832
- type: REQUEST,
833
- requestType: request.type,
834
- invalidateReason: _constants.INITIAL_BUILD
835
- });
836
- let {
837
- api,
838
- subRequestContentKeys
839
- } = this.createAPI(requestNodeId, previousInvalidations);
840
- try {
841
- let node = this.graph.getRequestNode(requestNodeId);
842
- this.stats.set(request.type, (this.stats.get(request.type) ?? 0) + 1);
843
- let result = await request.run({
844
- input: request.input,
845
- api,
846
- farm: this.farm,
847
- invalidateReason: node.invalidateReason,
848
- options: this.options,
849
- rustAtlaspack: this.rustAtlaspack
850
- });
851
- (0, _utils2.assertSignalNotAborted)(this.signal);
852
- this.completeRequest(requestNodeId);
853
- deferred.resolve(true);
854
- return result;
855
- } catch (err) {
856
- if (!(err instanceof _utils2.BuildAbortError) && request.type === requestTypes.dev_dep_request) {
857
- _logger().default.verbose({
858
- origin: '@atlaspack/core',
859
- message: `Failed DevDepRequest`,
860
- meta: {
861
- trackableEvent: 'failed_dev_dep_request',
862
- hasKey,
863
- hasValidResult
864
- }
865
- });
866
- }
867
- this.rejectRequest(requestNodeId);
868
- deferred.resolve(false);
869
- throw err;
870
- } finally {
871
- this.graph.replaceSubrequests(requestNodeId, [...subRequestContentKeys]);
872
- }
873
- }
874
- flushStats() {
875
- let requestTypeEntries = {};
876
- for (let key of Object.keys(requestTypes)) {
877
- requestTypeEntries[requestTypes[key]] = key;
878
- }
879
- let formattedStats = {};
880
- for (let [requestType, count] of this.stats.entries()) {
881
- let requestTypeName = requestTypeEntries[requestType];
882
- formattedStats[requestTypeName] = count;
883
- }
884
- this.stats = new Map();
885
- return formattedStats;
886
- }
887
- createAPI(requestId, previousInvalidations) {
888
- let subRequestContentKeys = new Set();
889
- return {
890
- api: {
891
- invalidateOnFileCreate: input => this.graph.invalidateOnFileCreate(requestId, input),
892
- invalidateOnConfigKeyChange: (filePath, configKey, contentHash) => this.graph.invalidateOnConfigKeyChange(requestId, filePath, configKey, contentHash),
893
- invalidateOnFileDelete: filePath => this.graph.invalidateOnFileDelete(requestId, filePath),
894
- invalidateOnFileUpdate: filePath => this.graph.invalidateOnFileUpdate(requestId, filePath),
895
- invalidateOnStartup: () => this.graph.invalidateOnStartup(requestId),
896
- invalidateOnBuild: () => this.graph.invalidateOnBuild(requestId),
897
- invalidateOnEnvChange: env => this.graph.invalidateOnEnvChange(requestId, env, this.options.env[env]),
898
- invalidateOnOptionChange: option => this.graph.invalidateOnOptionChange(requestId, option, this.options[option]),
899
- getInvalidations: () => previousInvalidations,
900
- storeResult: (result, cacheKey) => {
901
- this.storeResult(requestId, result, cacheKey);
902
- },
903
- getSubRequests: () => this.graph.getSubRequests(requestId),
904
- getInvalidSubRequests: () => this.graph.getInvalidSubRequests(requestId),
905
- getPreviousResult: ifMatch => {
906
- var _this$graph$getNode;
907
- let contentKey = (0, _nullthrows().default)((_this$graph$getNode = this.graph.getNode(requestId)) === null || _this$graph$getNode === void 0 ? void 0 : _this$graph$getNode.id);
908
- return this.getRequestResult(contentKey, ifMatch);
909
- },
910
- getRequestResult: id => this.getRequestResult(id),
911
- canSkipSubrequest: contentKey => {
912
- if (this.graph.hasContentKey(contentKey) && this.hasValidResult(this.graph.getNodeIdByContentKey(contentKey))) {
913
- subRequestContentKeys.add(contentKey);
914
- return true;
915
- }
916
- return false;
917
- },
918
- runRequest: (subRequest, opts) => {
919
- subRequestContentKeys.add(subRequest.id);
920
- return this.runRequest(subRequest, opts);
921
- }
922
- },
923
- subRequestContentKeys
924
- };
925
- }
926
- async writeToCache(signal) {
927
- let cacheKey = getCacheKey(this.options);
928
- let requestGraphKey = `requestGraph-${cacheKey}`;
929
- if (this.options.shouldDisableCache) {
930
- return;
931
- }
932
- let total = 0;
933
- (0, _ReporterRunner.report)({
934
- type: 'cache',
935
- phase: 'start',
936
- total,
937
- size: this.graph.nodes.length
938
- });
939
- let serialisedGraph = this.graph.serialize();
940
-
941
- // Delete an existing request graph cache, to prevent invalid states
942
- await this.options.cache.deleteLargeBlob(requestGraphKey);
943
- const serialiseAndSet = async (key, contents) => {
944
- if (signal !== null && signal !== void 0 && signal.aborted) {
945
- throw new Error('Serialization was aborted');
946
- }
947
- await this.options.cache.setLargeBlob(key, (0, _buildCache().serialize)(contents), signal ? {
948
- signal: signal
949
- } : undefined);
950
- total += 1;
951
- (0, _ReporterRunner.report)({
952
- type: 'cache',
953
- phase: 'write',
954
- total,
955
- size: this.graph.nodes.length
956
- });
957
- };
958
- let queue = new (_utils().PromiseQueue)({
959
- maxConcurrent: 32
960
- });
961
-
962
- // Preallocating a sparse array is faster than pushing when N is high enough
963
- let cacheableNodes = new Array(serialisedGraph.nodes.length);
964
- for (let i = 0; i < serialisedGraph.nodes.length; i += 1) {
965
- let node = serialisedGraph.nodes[i];
966
- let resultCacheKey = node === null || node === void 0 ? void 0 : node.resultCacheKey;
967
- if ((node === null || node === void 0 ? void 0 : node.type) === REQUEST && resultCacheKey != null && (node === null || node === void 0 ? void 0 : node.result) != null) {
968
- queue.add(() => serialiseAndSet(resultCacheKey, node.result));
969
-
970
- // eslint-disable-next-line no-unused-vars
971
- let {
972
- result: _,
973
- ...newNode
974
- } = node;
975
- cacheableNodes[i] = newNode;
976
- } else {
977
- cacheableNodes[i] = node;
978
- }
979
- }
980
- let nodeCountsPerBlob = [];
981
- for (let i = 0; i * this.graph.nodesPerBlob < cacheableNodes.length; i += 1) {
982
- let nodesStartIndex = i * this.graph.nodesPerBlob;
983
- let nodesEndIndex = Math.min((i + 1) * this.graph.nodesPerBlob, cacheableNodes.length);
984
- nodeCountsPerBlob.push(nodesEndIndex - nodesStartIndex);
985
- if (!this.graph.hasCachedRequestChunk(i)) {
986
- // We assume the request graph nodes are immutable and won't change
987
- let nodesToCache = cacheableNodes.slice(nodesStartIndex, nodesEndIndex);
988
- queue.add(() => serialiseAndSet(getRequestGraphNodeKey(i, cacheKey), nodesToCache).then(() => {
989
- // Succeeded in writing to disk, save that we have completed this chunk
990
- this.graph.setCachedRequestChunk(i);
991
- }));
992
- }
993
- }
994
- try {
995
- await queue.run();
996
-
997
- // Set the request graph after the queue is flushed to avoid writing an invalid state
998
- await serialiseAndSet(requestGraphKey, {
999
- ...serialisedGraph,
1000
- nodeCountsPerBlob,
1001
- nodes: undefined
1002
- });
1003
- let opts = getWatcherOptions(this.options);
1004
- let snapshotPath = _path2().default.join(this.options.cacheDir, `snapshot-${cacheKey}` + '.txt');
1005
- await this.options.outputFS.writeSnapshot(this.options.watchDir, snapshotPath, opts);
1006
- } catch (err) {
1007
- // If we have aborted, ignore the error and continue
1008
- if (!(signal !== null && signal !== void 0 && signal.aborted)) throw err;
1009
- }
1010
- (0, _ReporterRunner.report)({
1011
- type: 'cache',
1012
- phase: 'end',
1013
- total,
1014
- size: this.graph.nodes.length
1015
- });
1016
- }
1017
- static async init({
1018
- farm,
1019
- options,
1020
- rustAtlaspack
1021
- }) {
1022
- let graph = await loadRequestGraph(options);
1023
- return new RequestTracker({
1024
- farm,
1025
- graph,
1026
- options,
1027
- rustAtlaspack
1028
- });
1029
- }
1030
- }
1031
- exports.default = RequestTracker;
1032
- function getWatcherOptions({
1033
- watchIgnore = [],
1034
- cacheDir,
1035
- watchDir,
1036
- watchBackend
1037
- }) {
1038
- const uniqueDirs = [...new Set([...watchIgnore, ...['.git', '.hg'], cacheDir])];
1039
- const ignore = uniqueDirs.map(dir => _path2().default.resolve(watchDir, dir));
1040
- return {
1041
- ignore,
1042
- backend: watchBackend
1043
- };
1044
- }
1045
- function getCacheKey(options) {
1046
- return (0, _rust().hashString)(`${_constants.ATLASPACK_VERSION}:${JSON.stringify(options.entries)}:${options.mode}:${options.shouldBuildLazily ? 'lazy' : 'eager'}:${options.watchBackend ?? ''}`);
1047
- }
1048
- function getRequestGraphNodeKey(index, cacheKey) {
1049
- return `requestGraph-nodes-${index}-${cacheKey}`;
1050
- }
1051
- async function readAndDeserializeRequestGraph(cache, requestGraphKey, cacheKey) {
1052
- let bufferLength = 0;
1053
- const getAndDeserialize = async key => {
1054
- let buffer = await cache.getLargeBlob(key);
1055
- bufferLength += Buffer.byteLength(buffer);
1056
- return (0, _buildCache().deserialize)(buffer);
1057
- };
1058
- let serializedRequestGraph = await getAndDeserialize(requestGraphKey);
1059
- let nodePromises = serializedRequestGraph.nodeCountsPerBlob.map(async (nodesCount, i) => {
1060
- let nodes = await getAndDeserialize(getRequestGraphNodeKey(i, cacheKey));
1061
- _assert().default.equal(nodes.length, nodesCount, 'RequestTracker node chunk: invalid node count');
1062
- return nodes;
1063
- });
1064
- return {
1065
- requestGraph: RequestGraph.deserialize({
1066
- ...serializedRequestGraph,
1067
- nodes: (await Promise.all(nodePromises)).flat()
1068
- }),
1069
- // This is used inside atlaspack query for `.inspectCache`
1070
- bufferLength
1071
- };
1072
- }
1073
- async function loadRequestGraph(options) {
1074
- if (options.shouldDisableCache) {
1075
- return new RequestGraph();
1076
- }
1077
- let cacheKey = getCacheKey(options);
1078
- let requestGraphKey = `requestGraph-${cacheKey}`;
1079
- let timeout;
1080
- const snapshotKey = `snapshot-${cacheKey}`;
1081
- const snapshotPath = _path2().default.join(options.cacheDir, snapshotKey + '.txt');
1082
- _logger().default.verbose({
1083
- origin: '@atlaspack/core',
1084
- message: 'Loading request graph',
1085
- meta: {
1086
- cacheKey,
1087
- snapshotKey
1088
- }
1089
- });
1090
- if (await options.cache.hasLargeBlob(requestGraphKey)) {
1091
- try {
1092
- let {
1093
- requestGraph
1094
- } = await readAndDeserializeRequestGraph(options.cache, requestGraphKey, cacheKey);
1095
- let opts = getWatcherOptions(options);
1096
- timeout = setTimeout(() => {
1097
- _logger().default.warn({
1098
- origin: '@atlaspack/core',
1099
- message: `Retrieving file system events since last build...\nThis can take upto a minute after branch changes or npm/yarn installs.`
1100
- });
1101
- }, 5000);
1102
- let startTime = Date.now();
1103
- let events = await options.inputFS.getEventsSince(options.watchDir, snapshotPath, opts);
1104
- clearTimeout(timeout);
1105
- _logger().default.verbose({
1106
- origin: '@atlaspack/core',
1107
- message: `File system event count: ${events.length}`,
1108
- meta: {
1109
- trackableEvent: 'watcher_events_count',
1110
- watcherEventCount: events.length,
1111
- duration: Date.now() - startTime
1112
- }
1113
- });
1114
- requestGraph.invalidateUnpredictableNodes();
1115
- requestGraph.invalidateOnBuildNodes();
1116
- requestGraph.invalidateEnvNodes(options.env);
1117
- requestGraph.invalidateOptionNodes(options);
1118
- await requestGraph.respondToFSEvents(options.unstableFileInvalidations || events, options, 10000, true);
1119
- return requestGraph;
1120
- } catch (e) {
1121
- // Prevent logging fs events took too long warning
1122
- clearTimeout(timeout);
1123
- logErrorOnBailout(options, snapshotPath, e);
1124
- // This error means respondToFSEvents timed out handling the invalidation events
1125
- // In this case we'll return a fresh RequestGraph
1126
- return new RequestGraph();
1127
- }
1128
- }
1129
- _logger().default.verbose({
1130
- origin: '@atlaspack/core',
1131
- message: 'Cache entry for request tracker was not found, initializing a clean cache.',
1132
- meta: {
1133
- cacheKey,
1134
- snapshotKey
1135
- }
1136
- });
1137
- return new RequestGraph();
1138
- }
1139
- function logErrorOnBailout(options, snapshotPath, e) {
1140
- if (e.message && e.message.includes('invalid clockspec')) {
1141
- const snapshotContents = options.inputFS.readFileSync(snapshotPath, 'utf-8');
1142
- _logger().default.warn({
1143
- origin: '@atlaspack/core',
1144
- message: `Error reading clockspec from snapshot, building with clean cache.`,
1145
- meta: {
1146
- snapshotContents: snapshotContents,
1147
- trackableEvent: 'invalid_clockspec_error'
1148
- }
1149
- });
1150
- } else {
1151
- _logger().default.warn({
1152
- origin: '@atlaspack/core',
1153
- message: `Unexpected error loading cache from disk, building with clean cache.`,
1154
- meta: {
1155
- errorMessage: e.message,
1156
- errorStack: e.stack,
1157
- trackableEvent: 'cache_load_error'
1158
- }
1159
- });
1160
- }
1161
- }
1162
- function cleanUpOrphans(graph) {
1163
- if (graph.rootNodeId == null) {
1164
- return [];
1165
- }
1166
- const reachableNodes = new Set();
1167
- graph.traverse(nodeId => {
1168
- reachableNodes.add(nodeId);
1169
- });
1170
- const removedNodeIds = [];
1171
- graph.nodes.forEach((_node, nodeId) => {
1172
- if (!reachableNodes.has(nodeId)) {
1173
- removedNodeIds.push(nodeId);
1174
- graph.removeNode(nodeId);
1175
- }
1176
- });
1177
- return removedNodeIds;
1178
- }