@atlaspack/core 2.17.1 → 2.17.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/lib/Atlaspack.js +7 -1
- package/lib/RequestTracker.js +124 -11
- package/package.json +18 -14
- package/src/Atlaspack.js +8 -1
- package/src/RequestTracker.js +216 -20
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,47 @@
|
|
|
1
1
|
# @atlaspack/core
|
|
2
2
|
|
|
3
|
+
## 2.17.3
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- [#551](https://github.com/atlassian-labs/atlaspack/pull/551) [`30f6017`](https://github.com/atlassian-labs/atlaspack/commit/30f60175ba4d272c5fc193973c63bc298584775b) Thanks [@yamadapc](https://github.com/yamadapc)! - Log request tracker invalidation counts on start-up
|
|
8
|
+
|
|
9
|
+
- [#550](https://github.com/atlassian-labs/atlaspack/pull/550) [`3a3e8e7`](https://github.com/atlassian-labs/atlaspack/commit/3a3e8e7be9e2dffd7304436d792f0f595d59665a) Thanks [@yamadapc](https://github.com/yamadapc)! - Fix typescript declaration files
|
|
10
|
+
|
|
11
|
+
- [#555](https://github.com/atlassian-labs/atlaspack/pull/555) [`15c1e3c`](https://github.com/atlassian-labs/atlaspack/commit/15c1e3c0628bae4c768d76cf3afc53d6d0d7ce7c) Thanks [@alshdavid](https://github.com/alshdavid)! - Added ATLASPACK_NATIVE_THREADS env variable to control the number of threads used by the native thread schedular
|
|
12
|
+
|
|
13
|
+
- Updated dependencies [[`30f6017`](https://github.com/atlassian-labs/atlaspack/commit/30f60175ba4d272c5fc193973c63bc298584775b), [`3a3e8e7`](https://github.com/atlassian-labs/atlaspack/commit/3a3e8e7be9e2dffd7304436d792f0f595d59665a), [`1ab0a27`](https://github.com/atlassian-labs/atlaspack/commit/1ab0a275aeca40350415e2b03e7440d1dddc6228), [`b8a4ae8`](https://github.com/atlassian-labs/atlaspack/commit/b8a4ae8f83dc0a83d8b145c5f729936ce52080a3)]:
|
|
14
|
+
- @atlaspack/feature-flags@2.15.1
|
|
15
|
+
- @atlaspack/fs@2.15.3
|
|
16
|
+
- @atlaspack/rust@3.3.3
|
|
17
|
+
- @atlaspack/cache@3.2.3
|
|
18
|
+
- @atlaspack/graph@3.4.6
|
|
19
|
+
- @atlaspack/utils@2.14.8
|
|
20
|
+
- @atlaspack/package-manager@2.14.8
|
|
21
|
+
- @atlaspack/logger@2.14.8
|
|
22
|
+
- @atlaspack/profiler@2.14.6
|
|
23
|
+
- @atlaspack/types@2.14.8
|
|
24
|
+
- @atlaspack/workers@2.14.8
|
|
25
|
+
- @atlaspack/plugin@2.14.8
|
|
26
|
+
|
|
27
|
+
## 2.17.2
|
|
28
|
+
|
|
29
|
+
### Patch Changes
|
|
30
|
+
|
|
31
|
+
- Updated dependencies [[`a1773d2`](https://github.com/atlassian-labs/atlaspack/commit/a1773d2a62d0ef7805ac7524621dcabcc1afe929), [`556d6ab`](https://github.com/atlassian-labs/atlaspack/commit/556d6ab8ede759fa7f37fcd3f4da336ef1c55e8f)]:
|
|
32
|
+
- @atlaspack/feature-flags@2.15.0
|
|
33
|
+
- @atlaspack/logger@2.14.7
|
|
34
|
+
- @atlaspack/rust@3.3.2
|
|
35
|
+
- @atlaspack/cache@3.2.2
|
|
36
|
+
- @atlaspack/fs@2.15.2
|
|
37
|
+
- @atlaspack/graph@3.4.5
|
|
38
|
+
- @atlaspack/utils@2.14.7
|
|
39
|
+
- @atlaspack/package-manager@2.14.7
|
|
40
|
+
- @atlaspack/workers@2.14.7
|
|
41
|
+
- @atlaspack/profiler@2.14.5
|
|
42
|
+
- @atlaspack/types@2.14.7
|
|
43
|
+
- @atlaspack/plugin@2.14.7
|
|
44
|
+
|
|
3
45
|
## 2.17.1
|
|
4
46
|
|
|
5
47
|
### Patch Changes
|
package/lib/Atlaspack.js
CHANGED
|
@@ -207,10 +207,16 @@ class Atlaspack {
|
|
|
207
207
|
// $FlowFixMe
|
|
208
208
|
const version = require('../package.json').version;
|
|
209
209
|
await lmdb.put('current_session_version', Buffer.from(version));
|
|
210
|
+
let threads = undefined;
|
|
211
|
+
if (process.env.ATLASPACK_NATIVE_THREADS !== undefined) {
|
|
212
|
+
threads = parseInt(process.env.ATLASPACK_NATIVE_THREADS, 10);
|
|
213
|
+
} else if (process.env.NODE_ENV === 'test') {
|
|
214
|
+
threads = 2;
|
|
215
|
+
}
|
|
210
216
|
rustAtlaspack = await _atlaspackV.AtlaspackV3.create({
|
|
211
217
|
...options,
|
|
212
218
|
corePath: _path().default.join(__dirname, '..'),
|
|
213
|
-
threads
|
|
219
|
+
threads,
|
|
214
220
|
entries: Array.isArray(entries) ? entries : entries == null ? undefined : [entries],
|
|
215
221
|
env: resolvedOptions.env,
|
|
216
222
|
fs: inputFS && new _atlaspackV.FileSystemV3(inputFS),
|
package/lib/RequestTracker.js
CHANGED
|
@@ -279,6 +279,12 @@ class RequestGraph extends _graph().ContentGraph {
|
|
|
279
279
|
// If the node is invalidated, the cached request chunk on disk needs to be re-written
|
|
280
280
|
this.removeCachedRequestChunkForNode(nodeId);
|
|
281
281
|
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* Nodes that are invalidated on start-up, such as JavaScript babel configuration files which are
|
|
285
|
+
* imported when the build kicks-off and might doing arbitrary work such as reading from the file
|
|
286
|
+
* system.
|
|
287
|
+
*/
|
|
282
288
|
invalidateUnpredictableNodes() {
|
|
283
289
|
for (let nodeId of this.unpredicatableNodeIds) {
|
|
284
290
|
let node = (0, _nullthrows().default)(this.getNode(nodeId));
|
|
@@ -286,6 +292,10 @@ class RequestGraph extends _graph().ContentGraph {
|
|
|
286
292
|
this.invalidateNode(nodeId, _constants.STARTUP);
|
|
287
293
|
}
|
|
288
294
|
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Effectively uncacheable nodes.
|
|
298
|
+
*/
|
|
289
299
|
invalidateOnBuildNodes() {
|
|
290
300
|
for (let nodeId of this.invalidateOnBuildNodeIds) {
|
|
291
301
|
let node = (0, _nullthrows().default)(this.getNode(nodeId));
|
|
@@ -293,29 +303,45 @@ class RequestGraph extends _graph().ContentGraph {
|
|
|
293
303
|
this.invalidateNode(nodeId, _constants.STARTUP);
|
|
294
304
|
}
|
|
295
305
|
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* Nodes invalidated by environment changes, corresponds to `env: ...` inputs.
|
|
309
|
+
*/
|
|
296
310
|
invalidateEnvNodes(env) {
|
|
311
|
+
const invalidatedKeys = [];
|
|
297
312
|
for (let nodeId of this.envNodeIds) {
|
|
298
313
|
let node = (0, _nullthrows().default)(this.getNode(nodeId));
|
|
299
314
|
(0, _assert().default)(node.type === ENV);
|
|
300
|
-
|
|
315
|
+
const key = keyFromEnvContentKey(node.id);
|
|
316
|
+
if (env[key] !== node.value) {
|
|
317
|
+
invalidatedKeys.push(key);
|
|
301
318
|
let parentNodes = this.getNodeIdsConnectedTo(nodeId, requestGraphEdgeTypes.invalidated_by_update);
|
|
302
319
|
for (let parentNode of parentNodes) {
|
|
303
320
|
this.invalidateNode(parentNode, _constants.ENV_CHANGE);
|
|
304
321
|
}
|
|
305
322
|
}
|
|
306
323
|
}
|
|
324
|
+
return invalidatedKeys;
|
|
307
325
|
}
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* Nodes invalidated by option changes.
|
|
329
|
+
*/
|
|
308
330
|
invalidateOptionNodes(options) {
|
|
331
|
+
const invalidatedKeys = [];
|
|
309
332
|
for (let nodeId of this.optionNodeIds) {
|
|
310
333
|
let node = (0, _nullthrows().default)(this.getNode(nodeId));
|
|
311
334
|
(0, _assert().default)(node.type === OPTION);
|
|
312
|
-
|
|
335
|
+
const key = keyFromOptionContentKey(node.id);
|
|
336
|
+
if ((0, _utils2.hashFromOption)(options[key]) !== node.hash) {
|
|
337
|
+
invalidatedKeys.push(key);
|
|
313
338
|
let parentNodes = this.getNodeIdsConnectedTo(nodeId, requestGraphEdgeTypes.invalidated_by_update);
|
|
314
339
|
for (let parentNode of parentNodes) {
|
|
315
340
|
this.invalidateNode(parentNode, _constants.OPTION_CHANGE);
|
|
316
341
|
}
|
|
317
342
|
}
|
|
318
343
|
}
|
|
344
|
+
return invalidatedKeys;
|
|
319
345
|
}
|
|
320
346
|
invalidateOnConfigKeyChange(requestNodeId, filePath, configKey, contentHash) {
|
|
321
347
|
let configKeyNodeId = this.addNode(nodeFromConfigKey(filePath, configKey, contentHash));
|
|
@@ -1119,12 +1145,22 @@ async function loadRequestGraph(options) {
|
|
|
1119
1145
|
let timeout;
|
|
1120
1146
|
const snapshotKey = `snapshot-${cacheKey}`;
|
|
1121
1147
|
const snapshotPath = _path2().default.join(options.cacheDir, snapshotKey + '.txt');
|
|
1148
|
+
const commonMeta = {
|
|
1149
|
+
cacheKey,
|
|
1150
|
+
snapshotKey,
|
|
1151
|
+
cacheKeyOptions: {
|
|
1152
|
+
version: _constants.ATLASPACK_VERSION,
|
|
1153
|
+
entries: options.entries,
|
|
1154
|
+
mode: options.mode,
|
|
1155
|
+
shouldBuildLazily: options.shouldBuildLazily,
|
|
1156
|
+
watchBackend: options.watchBackend
|
|
1157
|
+
}
|
|
1158
|
+
};
|
|
1122
1159
|
_logger().default.verbose({
|
|
1123
1160
|
origin: '@atlaspack/core',
|
|
1124
1161
|
message: 'Loading request graph',
|
|
1125
1162
|
meta: {
|
|
1126
|
-
|
|
1127
|
-
snapshotKey
|
|
1163
|
+
...commonMeta
|
|
1128
1164
|
}
|
|
1129
1165
|
});
|
|
1130
1166
|
if (await options.cache.hasLargeBlob(requestGraphKey)) {
|
|
@@ -1146,16 +1182,29 @@ async function loadRequestGraph(options) {
|
|
|
1146
1182
|
origin: '@atlaspack/core',
|
|
1147
1183
|
message: `File system event count: ${events.length}`,
|
|
1148
1184
|
meta: {
|
|
1185
|
+
...commonMeta,
|
|
1149
1186
|
trackableEvent: 'watcher_events_count',
|
|
1150
1187
|
watcherEventCount: events.length,
|
|
1151
1188
|
duration: Date.now() - startTime
|
|
1152
1189
|
}
|
|
1153
1190
|
});
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1191
|
+
if ((0, _featureFlags().getFeatureFlag)('verboseRequestInvalidationStats')) {
|
|
1192
|
+
const invalidationStats = await invalidateRequestGraph(requestGraph, options, events);
|
|
1193
|
+
_logger().default.verbose({
|
|
1194
|
+
origin: '@atlaspack/core',
|
|
1195
|
+
message: 'Request track loaded from cache',
|
|
1196
|
+
meta: {
|
|
1197
|
+
...commonMeta,
|
|
1198
|
+
trackableEvent: 'request_tracker_cache_key_hit',
|
|
1199
|
+
invalidationStats
|
|
1200
|
+
}
|
|
1201
|
+
});
|
|
1202
|
+
} else {
|
|
1203
|
+
requestGraph.invalidateUnpredictableNodes();
|
|
1204
|
+
requestGraph.invalidateOnBuildNodes();
|
|
1205
|
+
requestGraph.invalidateEnvNodes(options.env);
|
|
1206
|
+
requestGraph.invalidateOptionNodes(options);
|
|
1207
|
+
}
|
|
1159
1208
|
return requestGraph;
|
|
1160
1209
|
} catch (e) {
|
|
1161
1210
|
// Prevent logging fs events took too long warning
|
|
@@ -1170,12 +1219,76 @@ async function loadRequestGraph(options) {
|
|
|
1170
1219
|
origin: '@atlaspack/core',
|
|
1171
1220
|
message: 'Cache entry for request tracker was not found, initializing a clean cache.',
|
|
1172
1221
|
meta: {
|
|
1173
|
-
|
|
1174
|
-
|
|
1222
|
+
...commonMeta,
|
|
1223
|
+
trackableEvent: 'request_tracker_cache_key_miss'
|
|
1175
1224
|
}
|
|
1176
1225
|
});
|
|
1177
1226
|
return new RequestGraph();
|
|
1178
1227
|
}
|
|
1228
|
+
|
|
1229
|
+
/**
|
|
1230
|
+
* A wrapper around an invalidation type / method
|
|
1231
|
+
*/
|
|
1232
|
+
|
|
1233
|
+
/**
|
|
1234
|
+
* Information about a certain cache invalidation type.
|
|
1235
|
+
*/
|
|
1236
|
+
|
|
1237
|
+
/**
|
|
1238
|
+
* Respond to unpredictable, build, environment changes, option changes and file-system events
|
|
1239
|
+
* invalidating RequestGraph nodes.
|
|
1240
|
+
*
|
|
1241
|
+
* Returns the count of nodes invalidated by each invalidation type.
|
|
1242
|
+
*/
|
|
1243
|
+
async function invalidateRequestGraph(requestGraph, options, events) {
|
|
1244
|
+
const invalidationFns = [{
|
|
1245
|
+
key: 'unpredictable',
|
|
1246
|
+
fn: () => requestGraph.invalidateUnpredictableNodes()
|
|
1247
|
+
}, {
|
|
1248
|
+
key: 'onBuild',
|
|
1249
|
+
fn: () => requestGraph.invalidateOnBuildNodes()
|
|
1250
|
+
}, {
|
|
1251
|
+
key: 'env',
|
|
1252
|
+
fn: () => requestGraph.invalidateEnvNodes(options.env)
|
|
1253
|
+
}, {
|
|
1254
|
+
key: 'option',
|
|
1255
|
+
fn: () => requestGraph.invalidateOptionNodes(options)
|
|
1256
|
+
}, {
|
|
1257
|
+
key: 'fsEvents',
|
|
1258
|
+
fn: async () => {
|
|
1259
|
+
await requestGraph.respondToFSEvents(options.unstableFileInvalidations || events, options, 10000, true);
|
|
1260
|
+
}
|
|
1261
|
+
}];
|
|
1262
|
+
const invalidations = [];
|
|
1263
|
+
for (const invalidation of invalidationFns) {
|
|
1264
|
+
invalidations.push(await runInvalidation(requestGraph, invalidation));
|
|
1265
|
+
}
|
|
1266
|
+
const invalidatedCount = invalidations.reduce((acc, invalidation) => acc + invalidation.count, 0);
|
|
1267
|
+
const requestCount = requestGraph.nodes.reduce((acc, node) => acc + ((node === null || node === void 0 ? void 0 : node.type) === REQUEST ? 1 : 0), 0);
|
|
1268
|
+
const nodeCount = requestGraph.nodes.length;
|
|
1269
|
+
return {
|
|
1270
|
+
invalidations,
|
|
1271
|
+
nodeCount,
|
|
1272
|
+
requestCount,
|
|
1273
|
+
invalidatedCount,
|
|
1274
|
+
nodeInvalidationRatio: invalidatedCount / nodeCount,
|
|
1275
|
+
requestInvalidationRatio: invalidatedCount / requestCount
|
|
1276
|
+
};
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1279
|
+
/**
|
|
1280
|
+
* Runs an invalidation function and reports metrics.
|
|
1281
|
+
*/
|
|
1282
|
+
async function runInvalidation(requestGraph, invalidationFn) {
|
|
1283
|
+
const startInvalidationCount = requestGraph.invalidNodeIds.size;
|
|
1284
|
+
const result = await invalidationFn.fn();
|
|
1285
|
+
const count = requestGraph.invalidNodeIds.size - startInvalidationCount;
|
|
1286
|
+
return {
|
|
1287
|
+
key: invalidationFn.key,
|
|
1288
|
+
count,
|
|
1289
|
+
changes: typeof result === 'object' && Array.isArray(result) ? result : null
|
|
1290
|
+
};
|
|
1291
|
+
}
|
|
1179
1292
|
function logErrorOnBailout(options, snapshotPath, e) {
|
|
1180
1293
|
if (e.message && e.message.includes('invalid clockspec')) {
|
|
1181
1294
|
const snapshotContents = options.inputFS.readFileSync(snapshotPath, 'utf-8');
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@atlaspack/core",
|
|
3
|
-
"version": "2.17.
|
|
3
|
+
"version": "2.17.3",
|
|
4
4
|
"license": "(MIT OR Apache-2.0)",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
"url": "https://github.com/atlassian-labs/atlaspack.git"
|
|
11
11
|
},
|
|
12
12
|
"main": "lib/index.js",
|
|
13
|
+
"types": "index.d.ts",
|
|
13
14
|
"source": "src/index.js",
|
|
14
15
|
"engines": {
|
|
15
16
|
"node": ">= 16.0.0"
|
|
@@ -22,20 +23,20 @@
|
|
|
22
23
|
"dependencies": {
|
|
23
24
|
"@mischnic/json-sourcemap": "^0.1.0",
|
|
24
25
|
"@atlaspack/build-cache": "2.13.3",
|
|
25
|
-
"@atlaspack/cache": "3.2.
|
|
26
|
+
"@atlaspack/cache": "3.2.3",
|
|
26
27
|
"@atlaspack/diagnostic": "2.14.1",
|
|
27
28
|
"@atlaspack/events": "2.14.1",
|
|
28
|
-
"@atlaspack/feature-flags": "2.
|
|
29
|
-
"@atlaspack/fs": "2.15.
|
|
30
|
-
"@atlaspack/graph": "3.4.
|
|
31
|
-
"@atlaspack/logger": "2.14.
|
|
32
|
-
"@atlaspack/package-manager": "2.14.
|
|
33
|
-
"@atlaspack/plugin": "2.14.
|
|
34
|
-
"@atlaspack/profiler": "2.14.
|
|
35
|
-
"@atlaspack/rust": "3.3.
|
|
36
|
-
"@atlaspack/types": "2.14.
|
|
37
|
-
"@atlaspack/utils": "2.14.
|
|
38
|
-
"@atlaspack/workers": "2.14.
|
|
29
|
+
"@atlaspack/feature-flags": "2.15.1",
|
|
30
|
+
"@atlaspack/fs": "2.15.3",
|
|
31
|
+
"@atlaspack/graph": "3.4.6",
|
|
32
|
+
"@atlaspack/logger": "2.14.8",
|
|
33
|
+
"@atlaspack/package-manager": "2.14.8",
|
|
34
|
+
"@atlaspack/plugin": "2.14.8",
|
|
35
|
+
"@atlaspack/profiler": "2.14.6",
|
|
36
|
+
"@atlaspack/rust": "3.3.3",
|
|
37
|
+
"@atlaspack/types": "2.14.8",
|
|
38
|
+
"@atlaspack/utils": "2.14.8",
|
|
39
|
+
"@atlaspack/workers": "2.14.8",
|
|
39
40
|
"@parcel/source-map": "^2.1.1",
|
|
40
41
|
"base-x": "^3.0.8",
|
|
41
42
|
"browserslist": "^4.6.6",
|
|
@@ -57,7 +58,10 @@
|
|
|
57
58
|
},
|
|
58
59
|
"exports": {
|
|
59
60
|
"./*": "./*",
|
|
60
|
-
".":
|
|
61
|
+
".": {
|
|
62
|
+
"types": "./index.d.ts",
|
|
63
|
+
"default": "./lib/index.js"
|
|
64
|
+
},
|
|
61
65
|
"./worker": {
|
|
62
66
|
"@atlaspack::sources": "./src/worker.js",
|
|
63
67
|
"default": "./lib/worker.js"
|
package/src/Atlaspack.js
CHANGED
|
@@ -165,10 +165,17 @@ export default class Atlaspack {
|
|
|
165
165
|
const version = require('../package.json').version;
|
|
166
166
|
await lmdb.put('current_session_version', Buffer.from(version));
|
|
167
167
|
|
|
168
|
+
let threads = undefined;
|
|
169
|
+
if (process.env.ATLASPACK_NATIVE_THREADS !== undefined) {
|
|
170
|
+
threads = parseInt(process.env.ATLASPACK_NATIVE_THREADS, 10);
|
|
171
|
+
} else if (process.env.NODE_ENV === 'test') {
|
|
172
|
+
threads = 2;
|
|
173
|
+
}
|
|
174
|
+
|
|
168
175
|
rustAtlaspack = await AtlaspackV3.create({
|
|
169
176
|
...options,
|
|
170
177
|
corePath: path.join(__dirname, '..'),
|
|
171
|
-
threads
|
|
178
|
+
threads,
|
|
172
179
|
entries: Array.isArray(entries)
|
|
173
180
|
? entries
|
|
174
181
|
: entries == null
|
package/src/RequestTracker.js
CHANGED
|
@@ -446,6 +446,11 @@ export class RequestGraph extends ContentGraph<
|
|
|
446
446
|
this.removeCachedRequestChunkForNode(nodeId);
|
|
447
447
|
}
|
|
448
448
|
|
|
449
|
+
/**
|
|
450
|
+
* Nodes that are invalidated on start-up, such as JavaScript babel configuration files which are
|
|
451
|
+
* imported when the build kicks-off and might doing arbitrary work such as reading from the file
|
|
452
|
+
* system.
|
|
453
|
+
*/
|
|
449
454
|
invalidateUnpredictableNodes() {
|
|
450
455
|
for (let nodeId of this.unpredicatableNodeIds) {
|
|
451
456
|
let node = nullthrows(this.getNode(nodeId));
|
|
@@ -454,6 +459,9 @@ export class RequestGraph extends ContentGraph<
|
|
|
454
459
|
}
|
|
455
460
|
}
|
|
456
461
|
|
|
462
|
+
/**
|
|
463
|
+
* Effectively uncacheable nodes.
|
|
464
|
+
*/
|
|
457
465
|
invalidateOnBuildNodes() {
|
|
458
466
|
for (let nodeId of this.invalidateOnBuildNodeIds) {
|
|
459
467
|
let node = nullthrows(this.getNode(nodeId));
|
|
@@ -462,11 +470,20 @@ export class RequestGraph extends ContentGraph<
|
|
|
462
470
|
}
|
|
463
471
|
}
|
|
464
472
|
|
|
465
|
-
|
|
473
|
+
/**
|
|
474
|
+
* Nodes invalidated by environment changes, corresponds to `env: ...` inputs.
|
|
475
|
+
*/
|
|
476
|
+
invalidateEnvNodes(env: EnvMap): string[] {
|
|
477
|
+
const invalidatedKeys = [];
|
|
478
|
+
|
|
466
479
|
for (let nodeId of this.envNodeIds) {
|
|
467
480
|
let node = nullthrows(this.getNode(nodeId));
|
|
468
481
|
invariant(node.type === ENV);
|
|
469
|
-
|
|
482
|
+
|
|
483
|
+
const key = keyFromEnvContentKey(node.id);
|
|
484
|
+
if (env[key] !== node.value) {
|
|
485
|
+
invalidatedKeys.push(key);
|
|
486
|
+
|
|
470
487
|
let parentNodes = this.getNodeIdsConnectedTo(
|
|
471
488
|
nodeId,
|
|
472
489
|
requestGraphEdgeTypes.invalidated_by_update,
|
|
@@ -476,15 +493,23 @@ export class RequestGraph extends ContentGraph<
|
|
|
476
493
|
}
|
|
477
494
|
}
|
|
478
495
|
}
|
|
496
|
+
|
|
497
|
+
return invalidatedKeys;
|
|
479
498
|
}
|
|
480
499
|
|
|
481
|
-
|
|
500
|
+
/**
|
|
501
|
+
* Nodes invalidated by option changes.
|
|
502
|
+
*/
|
|
503
|
+
invalidateOptionNodes(options: AtlaspackOptions): string[] {
|
|
504
|
+
const invalidatedKeys = [];
|
|
505
|
+
|
|
482
506
|
for (let nodeId of this.optionNodeIds) {
|
|
483
507
|
let node = nullthrows(this.getNode(nodeId));
|
|
484
508
|
invariant(node.type === OPTION);
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
) {
|
|
509
|
+
const key = keyFromOptionContentKey(node.id);
|
|
510
|
+
|
|
511
|
+
if (hashFromOption(options[key]) !== node.hash) {
|
|
512
|
+
invalidatedKeys.push(key);
|
|
488
513
|
let parentNodes = this.getNodeIdsConnectedTo(
|
|
489
514
|
nodeId,
|
|
490
515
|
requestGraphEdgeTypes.invalidated_by_update,
|
|
@@ -494,6 +519,8 @@ export class RequestGraph extends ContentGraph<
|
|
|
494
519
|
}
|
|
495
520
|
}
|
|
496
521
|
}
|
|
522
|
+
|
|
523
|
+
return invalidatedKeys;
|
|
497
524
|
}
|
|
498
525
|
|
|
499
526
|
invalidateOnConfigKeyChange(
|
|
@@ -1727,12 +1754,23 @@ async function loadRequestGraph(options): Async<RequestGraph> {
|
|
|
1727
1754
|
const snapshotKey = `snapshot-${cacheKey}`;
|
|
1728
1755
|
const snapshotPath = path.join(options.cacheDir, snapshotKey + '.txt');
|
|
1729
1756
|
|
|
1757
|
+
const commonMeta = {
|
|
1758
|
+
cacheKey,
|
|
1759
|
+
snapshotKey,
|
|
1760
|
+
cacheKeyOptions: {
|
|
1761
|
+
version: ATLASPACK_VERSION,
|
|
1762
|
+
entries: options.entries,
|
|
1763
|
+
mode: options.mode,
|
|
1764
|
+
shouldBuildLazily: options.shouldBuildLazily,
|
|
1765
|
+
watchBackend: options.watchBackend,
|
|
1766
|
+
},
|
|
1767
|
+
};
|
|
1768
|
+
|
|
1730
1769
|
logger.verbose({
|
|
1731
1770
|
origin: '@atlaspack/core',
|
|
1732
1771
|
message: 'Loading request graph',
|
|
1733
1772
|
meta: {
|
|
1734
|
-
|
|
1735
|
-
snapshotKey,
|
|
1773
|
+
...commonMeta,
|
|
1736
1774
|
},
|
|
1737
1775
|
});
|
|
1738
1776
|
|
|
@@ -1767,23 +1805,36 @@ async function loadRequestGraph(options): Async<RequestGraph> {
|
|
|
1767
1805
|
origin: '@atlaspack/core',
|
|
1768
1806
|
message: `File system event count: ${events.length}`,
|
|
1769
1807
|
meta: {
|
|
1808
|
+
...commonMeta,
|
|
1770
1809
|
trackableEvent: 'watcher_events_count',
|
|
1771
1810
|
watcherEventCount: events.length,
|
|
1772
1811
|
duration: Date.now() - startTime,
|
|
1773
1812
|
},
|
|
1774
1813
|
});
|
|
1775
1814
|
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1815
|
+
if (getFeatureFlag('verboseRequestInvalidationStats')) {
|
|
1816
|
+
const invalidationStats = await invalidateRequestGraph(
|
|
1817
|
+
requestGraph,
|
|
1818
|
+
options,
|
|
1819
|
+
events,
|
|
1820
|
+
);
|
|
1821
|
+
|
|
1822
|
+
logger.verbose({
|
|
1823
|
+
origin: '@atlaspack/core',
|
|
1824
|
+
message: 'Request track loaded from cache',
|
|
1825
|
+
meta: {
|
|
1826
|
+
...commonMeta,
|
|
1827
|
+
trackableEvent: 'request_tracker_cache_key_hit',
|
|
1828
|
+
invalidationStats,
|
|
1829
|
+
},
|
|
1830
|
+
});
|
|
1831
|
+
} else {
|
|
1832
|
+
requestGraph.invalidateUnpredictableNodes();
|
|
1833
|
+
requestGraph.invalidateOnBuildNodes();
|
|
1834
|
+
requestGraph.invalidateEnvNodes(options.env);
|
|
1835
|
+
requestGraph.invalidateOptionNodes(options);
|
|
1836
|
+
}
|
|
1780
1837
|
|
|
1781
|
-
await requestGraph.respondToFSEvents(
|
|
1782
|
-
options.unstableFileInvalidations || events,
|
|
1783
|
-
options,
|
|
1784
|
-
10000,
|
|
1785
|
-
true,
|
|
1786
|
-
);
|
|
1787
1838
|
return requestGraph;
|
|
1788
1839
|
} catch (e) {
|
|
1789
1840
|
// Prevent logging fs events took too long warning
|
|
@@ -1800,13 +1851,158 @@ async function loadRequestGraph(options): Async<RequestGraph> {
|
|
|
1800
1851
|
message:
|
|
1801
1852
|
'Cache entry for request tracker was not found, initializing a clean cache.',
|
|
1802
1853
|
meta: {
|
|
1803
|
-
|
|
1804
|
-
|
|
1854
|
+
...commonMeta,
|
|
1855
|
+
trackableEvent: 'request_tracker_cache_key_miss',
|
|
1805
1856
|
},
|
|
1806
1857
|
});
|
|
1807
1858
|
return new RequestGraph();
|
|
1808
1859
|
}
|
|
1809
1860
|
|
|
1861
|
+
/**
|
|
1862
|
+
* A wrapper around an invalidation type / method
|
|
1863
|
+
*/
|
|
1864
|
+
type InvalidationFn = {|
|
|
1865
|
+
key: string,
|
|
1866
|
+
fn: () => string[] | void | Promise<void>,
|
|
1867
|
+
|};
|
|
1868
|
+
|
|
1869
|
+
type InvalidationStats = {|
|
|
1870
|
+
/**
|
|
1871
|
+
* Total number of request graph nodes
|
|
1872
|
+
*/
|
|
1873
|
+
nodeCount: number,
|
|
1874
|
+
/**
|
|
1875
|
+
* Number of requests in RequestGraph
|
|
1876
|
+
*/
|
|
1877
|
+
requestCount: number,
|
|
1878
|
+
/**
|
|
1879
|
+
* Number of nodes that have been invalidated.
|
|
1880
|
+
*/
|
|
1881
|
+
invalidatedCount: number,
|
|
1882
|
+
/**
|
|
1883
|
+
* Percentage of requests that have been invalidated
|
|
1884
|
+
*/
|
|
1885
|
+
requestInvalidationRatio: number,
|
|
1886
|
+
/**
|
|
1887
|
+
* Percentage of nodes that have been invalidated
|
|
1888
|
+
*/
|
|
1889
|
+
nodeInvalidationRatio: number,
|
|
1890
|
+
/**
|
|
1891
|
+
* Details for each invalidation type
|
|
1892
|
+
*/
|
|
1893
|
+
invalidations: InvalidationFnStats[],
|
|
1894
|
+
|};
|
|
1895
|
+
|
|
1896
|
+
/**
|
|
1897
|
+
* Information about a certain cache invalidation type.
|
|
1898
|
+
*/
|
|
1899
|
+
type InvalidationFnStats = {|
|
|
1900
|
+
/**
|
|
1901
|
+
* Invalidation type, one of:
|
|
1902
|
+
*
|
|
1903
|
+
* - unpredictable
|
|
1904
|
+
* - onBuild
|
|
1905
|
+
* - env
|
|
1906
|
+
* - option
|
|
1907
|
+
* - fsEvents
|
|
1908
|
+
*/
|
|
1909
|
+
key: string,
|
|
1910
|
+
/**
|
|
1911
|
+
* Number of invalidated nodes coming from this invalidation type.
|
|
1912
|
+
*/
|
|
1913
|
+
count: number,
|
|
1914
|
+
/**
|
|
1915
|
+
* If this is a env or option invalidation, this key will contain the list of changed values.
|
|
1916
|
+
*/
|
|
1917
|
+
changes: null | string[],
|
|
1918
|
+
|};
|
|
1919
|
+
|
|
1920
|
+
/**
|
|
1921
|
+
* Respond to unpredictable, build, environment changes, option changes and file-system events
|
|
1922
|
+
* invalidating RequestGraph nodes.
|
|
1923
|
+
*
|
|
1924
|
+
* Returns the count of nodes invalidated by each invalidation type.
|
|
1925
|
+
*/
|
|
1926
|
+
async function invalidateRequestGraph(
|
|
1927
|
+
requestGraph: RequestGraph,
|
|
1928
|
+
options: AtlaspackOptions,
|
|
1929
|
+
events: Event[],
|
|
1930
|
+
): Promise<InvalidationStats> {
|
|
1931
|
+
const invalidationFns: InvalidationFn[] = [
|
|
1932
|
+
{
|
|
1933
|
+
key: 'unpredictable',
|
|
1934
|
+
fn: () => requestGraph.invalidateUnpredictableNodes(),
|
|
1935
|
+
},
|
|
1936
|
+
{
|
|
1937
|
+
key: 'onBuild',
|
|
1938
|
+
fn: () => requestGraph.invalidateOnBuildNodes(),
|
|
1939
|
+
},
|
|
1940
|
+
{
|
|
1941
|
+
key: 'env',
|
|
1942
|
+
fn: () => requestGraph.invalidateEnvNodes(options.env),
|
|
1943
|
+
},
|
|
1944
|
+
{
|
|
1945
|
+
key: 'option',
|
|
1946
|
+
fn: () => requestGraph.invalidateOptionNodes(options),
|
|
1947
|
+
},
|
|
1948
|
+
{
|
|
1949
|
+
key: 'fsEvents',
|
|
1950
|
+
fn: async () => {
|
|
1951
|
+
await requestGraph.respondToFSEvents(
|
|
1952
|
+
options.unstableFileInvalidations || events,
|
|
1953
|
+
options,
|
|
1954
|
+
10000,
|
|
1955
|
+
true,
|
|
1956
|
+
);
|
|
1957
|
+
},
|
|
1958
|
+
},
|
|
1959
|
+
];
|
|
1960
|
+
|
|
1961
|
+
const invalidations = [];
|
|
1962
|
+
for (const invalidation of invalidationFns) {
|
|
1963
|
+
invalidations.push(await runInvalidation(requestGraph, invalidation));
|
|
1964
|
+
}
|
|
1965
|
+
const invalidatedCount = invalidations.reduce(
|
|
1966
|
+
(acc, invalidation) => acc + invalidation.count,
|
|
1967
|
+
0,
|
|
1968
|
+
);
|
|
1969
|
+
const requestCount = requestGraph.nodes.reduce(
|
|
1970
|
+
(acc, node) => acc + (node?.type === REQUEST ? 1 : 0),
|
|
1971
|
+
0,
|
|
1972
|
+
);
|
|
1973
|
+
const nodeCount = requestGraph.nodes.length;
|
|
1974
|
+
const nodeInvalidationRatio = invalidatedCount / nodeCount;
|
|
1975
|
+
const requestInvalidationRatio = invalidatedCount / requestCount;
|
|
1976
|
+
|
|
1977
|
+
return {
|
|
1978
|
+
invalidations,
|
|
1979
|
+
nodeCount,
|
|
1980
|
+
requestCount,
|
|
1981
|
+
invalidatedCount,
|
|
1982
|
+
nodeInvalidationRatio,
|
|
1983
|
+
requestInvalidationRatio,
|
|
1984
|
+
};
|
|
1985
|
+
}
|
|
1986
|
+
|
|
1987
|
+
/**
|
|
1988
|
+
* Runs an invalidation function and reports metrics.
|
|
1989
|
+
*/
|
|
1990
|
+
async function runInvalidation(
|
|
1991
|
+
requestGraph: RequestGraph,
|
|
1992
|
+
invalidationFn: InvalidationFn,
|
|
1993
|
+
): Promise<InvalidationFnStats> {
|
|
1994
|
+
const startInvalidationCount = requestGraph.invalidNodeIds.size;
|
|
1995
|
+
const result = await invalidationFn.fn();
|
|
1996
|
+
const count = requestGraph.invalidNodeIds.size - startInvalidationCount;
|
|
1997
|
+
|
|
1998
|
+
return {
|
|
1999
|
+
key: invalidationFn.key,
|
|
2000
|
+
count,
|
|
2001
|
+
changes:
|
|
2002
|
+
typeof result === 'object' && Array.isArray(result) ? result : null,
|
|
2003
|
+
};
|
|
2004
|
+
}
|
|
2005
|
+
|
|
1810
2006
|
function logErrorOnBailout(
|
|
1811
2007
|
options: AtlaspackOptions,
|
|
1812
2008
|
snapshotPath: string,
|