@atlaspack/core 2.17.3 → 2.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/lib/AssetGraph.js +17 -6
- package/lib/Atlaspack.js +3 -1
- package/lib/BundleGraph.js +6 -5
- package/lib/Dependency.js +6 -2
- package/lib/Environment.js +4 -3
- package/lib/EnvironmentManager.js +137 -0
- package/lib/InternalConfig.js +3 -2
- package/lib/PackagerRunner.js +52 -15
- package/lib/RequestTracker.js +191 -89
- package/lib/UncommittedAsset.js +20 -2
- package/lib/applyRuntimes.js +2 -1
- package/lib/assetUtils.js +2 -1
- package/lib/atlaspack-v3/worker/worker.js +8 -0
- package/lib/public/Asset.js +3 -2
- package/lib/public/Bundle.js +2 -1
- package/lib/public/BundleGraph.js +21 -5
- package/lib/public/Config.js +98 -3
- package/lib/public/Dependency.js +2 -1
- package/lib/public/MutableBundleGraph.js +2 -1
- package/lib/public/Target.js +2 -1
- package/lib/requests/AssetGraphRequest.js +13 -1
- package/lib/requests/AssetRequest.js +2 -1
- package/lib/requests/BundleGraphRequest.js +13 -1
- package/lib/requests/ConfigRequest.js +27 -4
- package/lib/requests/TargetRequest.js +18 -16
- package/lib/requests/WriteBundleRequest.js +15 -3
- package/lib/requests/WriteBundlesRequest.js +1 -0
- package/lib/resolveOptions.js +4 -2
- package/package.json +13 -13
- package/src/AssetGraph.js +12 -6
- package/src/Atlaspack.js +5 -4
- package/src/BundleGraph.js +13 -8
- package/src/Dependency.js +13 -5
- package/src/Environment.js +8 -5
- package/src/EnvironmentManager.js +145 -0
- package/src/InternalConfig.js +6 -5
- package/src/PackagerRunner.js +72 -20
- package/src/RequestTracker.js +330 -146
- package/src/UncommittedAsset.js +23 -3
- package/src/applyRuntimes.js +6 -1
- package/src/assetUtils.js +4 -3
- package/src/atlaspack-v3/worker/compat/plugin-config.js +9 -5
- package/src/atlaspack-v3/worker/worker.js +7 -0
- package/src/public/Asset.js +9 -2
- package/src/public/Bundle.js +2 -1
- package/src/public/BundleGraph.js +22 -5
- package/src/public/Config.js +129 -14
- package/src/public/Dependency.js +2 -1
- package/src/public/MutableBundleGraph.js +2 -1
- package/src/public/Target.js +2 -1
- package/src/requests/AssetGraphRequest.js +13 -3
- package/src/requests/AssetRequest.js +2 -1
- package/src/requests/BundleGraphRequest.js +13 -3
- package/src/requests/ConfigRequest.js +33 -9
- package/src/requests/TargetRequest.js +19 -25
- package/src/requests/WriteBundleRequest.js +14 -8
- package/src/requests/WriteBundlesRequest.js +1 -0
- package/src/resolveOptions.js +4 -2
- package/src/types.js +9 -7
- package/test/Environment.test.js +43 -34
- package/test/EnvironmentManager.test.js +192 -0
- package/test/PublicEnvironment.test.js +10 -7
- package/test/RequestTracker.test.js +115 -3
- package/test/public/Config.test.js +108 -0
- package/test/requests/ConfigRequest.test.js +187 -3
- package/test/test-utils.js +4 -9
package/lib/RequestTracker.js
CHANGED
|
@@ -6,9 +6,13 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
6
6
|
exports.RequestGraph = void 0;
|
|
7
7
|
exports.cleanUpOrphans = cleanUpOrphans;
|
|
8
8
|
exports.default = void 0;
|
|
9
|
+
exports.getBiggestFSEventsInvalidations = getBiggestFSEventsInvalidations;
|
|
9
10
|
exports.getWatcherOptions = getWatcherOptions;
|
|
11
|
+
exports.invalidateRequestGraph = invalidateRequestGraph;
|
|
12
|
+
exports.invalidateRequestGraphFSEvents = invalidateRequestGraphFSEvents;
|
|
10
13
|
exports.readAndDeserializeRequestGraph = readAndDeserializeRequestGraph;
|
|
11
14
|
exports.requestTypes = exports.requestGraphEdgeTypes = void 0;
|
|
15
|
+
exports.runInvalidation = runInvalidation;
|
|
12
16
|
function _assert() {
|
|
13
17
|
const data = _interopRequireWildcard(require("assert"));
|
|
14
18
|
_assert = function () {
|
|
@@ -84,6 +88,14 @@ var _projectPath = require("./projectPath");
|
|
|
84
88
|
var _ReporterRunner = require("./ReporterRunner");
|
|
85
89
|
var _ConfigRequest = require("./requests/ConfigRequest");
|
|
86
90
|
var _utils2 = require("./utils");
|
|
91
|
+
function _perf_hooks() {
|
|
92
|
+
const data = require("perf_hooks");
|
|
93
|
+
_perf_hooks = function () {
|
|
94
|
+
return data;
|
|
95
|
+
};
|
|
96
|
+
return data;
|
|
97
|
+
}
|
|
98
|
+
var _EnvironmentManager = require("./EnvironmentManager");
|
|
87
99
|
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
|
88
100
|
function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
|
|
89
101
|
function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
|
|
@@ -151,7 +163,7 @@ const nodeFromOption = (option, value) => ({
|
|
|
151
163
|
hash: (0, _utils2.hashFromOption)(value)
|
|
152
164
|
});
|
|
153
165
|
const nodeFromConfigKey = (fileName, configKey, contentHash) => ({
|
|
154
|
-
id: `config_key:${(0, _projectPath.fromProjectPathRelative)(fileName)}:${configKey}`,
|
|
166
|
+
id: `config_key:${(0, _projectPath.fromProjectPathRelative)(fileName)}:${JSON.stringify(configKey)}`,
|
|
155
167
|
type: CONFIG_KEY,
|
|
156
168
|
configKey,
|
|
157
169
|
contentHash
|
|
@@ -434,8 +446,8 @@ class RequestGraph extends _graph().ContentGraph {
|
|
|
434
446
|
this.invalidateOnBuildNodeIds.add(requestNodeId);
|
|
435
447
|
}
|
|
436
448
|
invalidateOnEnvChange(requestNodeId, env, value) {
|
|
437
|
-
|
|
438
|
-
|
|
449
|
+
const envNode = nodeFromEnv(env, value);
|
|
450
|
+
const envNodeId = this.addNode(envNode);
|
|
439
451
|
if (!this.hasEdge(requestNodeId, envNodeId, requestGraphEdgeTypes.invalidated_by_update)) {
|
|
440
452
|
this.addEdge(requestNodeId, envNodeId, requestGraphEdgeTypes.invalidated_by_update);
|
|
441
453
|
}
|
|
@@ -582,10 +594,12 @@ class RequestGraph extends _graph().ContentGraph {
|
|
|
582
594
|
aboveCache.set(fileNameNodeId, above);
|
|
583
595
|
return above;
|
|
584
596
|
};
|
|
597
|
+
const invalidationsByPath = new Map();
|
|
585
598
|
for (let {
|
|
586
599
|
path: _path,
|
|
587
600
|
type
|
|
588
601
|
} of events) {
|
|
602
|
+
const invalidationsBefore = this.getInvalidNodeCount();
|
|
589
603
|
if (!enableOptimization && process.env.ATLASPACK_DISABLE_CACHE_TIMEOUT !== 'true' && ++count === 256) {
|
|
590
604
|
let duration = Date.now() - startTime;
|
|
591
605
|
predictedTime = duration * (events.length >> 8);
|
|
@@ -622,7 +636,10 @@ class RequestGraph extends _graph().ContentGraph {
|
|
|
622
636
|
this.invalidNodeIds.add(id);
|
|
623
637
|
}
|
|
624
638
|
}
|
|
625
|
-
return
|
|
639
|
+
return {
|
|
640
|
+
didInvalidate: true,
|
|
641
|
+
invalidationsByPath: new Map()
|
|
642
|
+
};
|
|
626
643
|
}
|
|
627
644
|
|
|
628
645
|
// sometimes mac os reports update events as create events.
|
|
@@ -679,10 +696,18 @@ class RequestGraph extends _graph().ContentGraph {
|
|
|
679
696
|
this.removeNode(nodeId, removeOrphans);
|
|
680
697
|
}
|
|
681
698
|
let configKeyNodes = this.configKeyNodes.get(_filePath);
|
|
682
|
-
|
|
699
|
+
|
|
700
|
+
// With granular invalidations we will always run this block,
|
|
701
|
+
// so even if we get a create event (for whatever reason), we will still
|
|
702
|
+
// try to limit invalidations from config key changes through hashing.
|
|
703
|
+
//
|
|
704
|
+
// Currently create events can invalidate a large number of nodes due to
|
|
705
|
+
// "create above" invalidations.
|
|
706
|
+
const isConfigKeyChange = (0, _featureFlags().getFeatureFlag)('granularTsConfigInvalidation') || type === 'delete' || type === 'update';
|
|
707
|
+
if (configKeyNodes && isConfigKeyChange) {
|
|
683
708
|
for (let nodeId of configKeyNodes) {
|
|
684
709
|
let isInvalid = type === 'delete';
|
|
685
|
-
if (type
|
|
710
|
+
if (type !== 'delete') {
|
|
686
711
|
let node = this.getNode(nodeId);
|
|
687
712
|
(0, _assert().default)(node && node.type === CONFIG_KEY);
|
|
688
713
|
let contentHash = await (0, _ConfigRequest.getConfigKeyContentHash)(_filePath, node.configKey, options);
|
|
@@ -697,6 +722,8 @@ class RequestGraph extends _graph().ContentGraph {
|
|
|
697
722
|
}
|
|
698
723
|
}
|
|
699
724
|
}
|
|
725
|
+
const invalidationsAfter = this.getInvalidNodeCount();
|
|
726
|
+
invalidationsByPath.set(_path, (invalidationsByPath.get(_path) ?? 0) + (invalidationsAfter - invalidationsBefore));
|
|
700
727
|
}
|
|
701
728
|
if ((0, _featureFlags().getFeatureFlag)('fixQuadraticCacheInvalidation')) {
|
|
702
729
|
cleanUpOrphans(this);
|
|
@@ -714,7 +741,10 @@ class RequestGraph extends _graph().ContentGraph {
|
|
|
714
741
|
numberOfInvalidatedNodes: invalidatedNodes.size
|
|
715
742
|
}
|
|
716
743
|
});
|
|
717
|
-
return
|
|
744
|
+
return {
|
|
745
|
+
didInvalidate,
|
|
746
|
+
invalidationsByPath
|
|
747
|
+
};
|
|
718
748
|
}
|
|
719
749
|
hasCachedRequestChunk(index) {
|
|
720
750
|
return this.cachedRequestChunks.has(index);
|
|
@@ -725,6 +755,13 @@ class RequestGraph extends _graph().ContentGraph {
|
|
|
725
755
|
removeCachedRequestChunkForNode(nodeId) {
|
|
726
756
|
this.cachedRequestChunks.delete(Math.floor(nodeId / this.nodesPerBlob));
|
|
727
757
|
}
|
|
758
|
+
|
|
759
|
+
/**
|
|
760
|
+
* Returns the number of invalidated nodes in the graph.
|
|
761
|
+
*/
|
|
762
|
+
getInvalidNodeCount() {
|
|
763
|
+
return this.invalidNodeIds.size;
|
|
764
|
+
}
|
|
728
765
|
}
|
|
729
766
|
exports.RequestGraph = RequestGraph;
|
|
730
767
|
class RequestTracker {
|
|
@@ -970,84 +1007,88 @@ class RequestTracker {
|
|
|
970
1007
|
return result;
|
|
971
1008
|
}
|
|
972
1009
|
}
|
|
973
|
-
await runCacheImprovements(async cache => {
|
|
974
|
-
await cache.getNativeRef().startWriteTransaction();
|
|
975
|
-
}, () => Promise.resolve());
|
|
976
1010
|
let cacheKey = getCacheKey(this.options);
|
|
977
|
-
let requestGraphKey = `requestGraph-${cacheKey}`;
|
|
1011
|
+
let requestGraphKey = (0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements') ? `${cacheKey}/RequestGraph` : `requestGraph-${cacheKey}`;
|
|
1012
|
+
let snapshotKey = (0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements') ? `${cacheKey}/snapshot` : `snapshot-${cacheKey}`;
|
|
978
1013
|
if (this.options.shouldDisableCache) {
|
|
979
1014
|
return;
|
|
980
1015
|
}
|
|
981
1016
|
let total = 0;
|
|
982
|
-
(
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
size: this.graph.nodes.length
|
|
987
|
-
});
|
|
988
|
-
let serialisedGraph = this.graph.serialize();
|
|
989
|
-
|
|
990
|
-
// Delete an existing request graph cache, to prevent invalid states
|
|
991
|
-
await this.options.cache.deleteLargeBlob(requestGraphKey);
|
|
992
|
-
const serialiseAndSet = async (key, contents) => {
|
|
993
|
-
if (signal !== null && signal !== void 0 && signal.aborted) {
|
|
994
|
-
throw new Error('Serialization was aborted');
|
|
995
|
-
}
|
|
996
|
-
await runCacheImprovements(cache => {
|
|
997
|
-
(0, _logger().instrument)(`cache.put(${key})`, () => {
|
|
998
|
-
cache.getNativeRef().putNoConfirm(key, (0, _buildCache().serialize)(contents));
|
|
999
|
-
});
|
|
1000
|
-
return Promise.resolve();
|
|
1001
|
-
}, async () => {
|
|
1002
|
-
await this.options.cache.setLargeBlob(key, (0, _buildCache().serialize)(contents), signal ? {
|
|
1003
|
-
signal: signal
|
|
1004
|
-
} : undefined);
|
|
1005
|
-
});
|
|
1006
|
-
total += 1;
|
|
1017
|
+
await runCacheImprovements(async cache => {
|
|
1018
|
+
await cache.getNativeRef().startWriteTransaction();
|
|
1019
|
+
}, () => Promise.resolve());
|
|
1020
|
+
try {
|
|
1007
1021
|
(0, _ReporterRunner.report)({
|
|
1008
1022
|
type: 'cache',
|
|
1009
|
-
phase: '
|
|
1023
|
+
phase: 'start',
|
|
1010
1024
|
total,
|
|
1011
1025
|
size: this.graph.nodes.length
|
|
1012
1026
|
});
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1027
|
+
if ((0, _featureFlags().getFeatureFlag)('environmentDeduplication')) {
|
|
1028
|
+
await (0, _EnvironmentManager.writeEnvironmentsToCache)(options.cache);
|
|
1029
|
+
}
|
|
1030
|
+
let serialisedGraph = this.graph.serialize();
|
|
1017
1031
|
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1032
|
+
// Delete an existing request graph cache, to prevent invalid states
|
|
1033
|
+
await this.options.cache.deleteLargeBlob(requestGraphKey);
|
|
1034
|
+
const serialiseAndSet = async (key, contents) => {
|
|
1035
|
+
if (signal !== null && signal !== void 0 && signal.aborted) {
|
|
1036
|
+
throw new Error('Serialization was aborted');
|
|
1037
|
+
}
|
|
1038
|
+
await runCacheImprovements(cache => {
|
|
1039
|
+
(0, _logger().instrument)(`RequestTracker::writeToCache::cache.put(${key})`, () => {
|
|
1040
|
+
cache.getNativeRef().putNoConfirm(key, (0, _buildCache().serialize)(contents));
|
|
1041
|
+
});
|
|
1042
|
+
return Promise.resolve();
|
|
1043
|
+
}, async () => {
|
|
1044
|
+
await this.options.cache.setLargeBlob(key, (0, _buildCache().serialize)(contents), signal ? {
|
|
1045
|
+
signal: signal
|
|
1046
|
+
} : undefined);
|
|
1047
|
+
});
|
|
1048
|
+
total += 1;
|
|
1049
|
+
(0, _ReporterRunner.report)({
|
|
1050
|
+
type: 'cache',
|
|
1051
|
+
phase: 'write',
|
|
1052
|
+
total,
|
|
1053
|
+
size: this.graph.nodes.length
|
|
1054
|
+
});
|
|
1055
|
+
};
|
|
1056
|
+
let queue = new (_utils().PromiseQueue)({
|
|
1057
|
+
maxConcurrent: 32
|
|
1058
|
+
});
|
|
1025
1059
|
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1060
|
+
// Preallocating a sparse array is faster than pushing when N is high enough
|
|
1061
|
+
let cacheableNodes = new Array(serialisedGraph.nodes.length);
|
|
1062
|
+
for (let i = 0; i < serialisedGraph.nodes.length; i += 1) {
|
|
1063
|
+
let node = serialisedGraph.nodes[i];
|
|
1064
|
+
let resultCacheKey = node === null || node === void 0 ? void 0 : node.resultCacheKey;
|
|
1065
|
+
if ((node === null || node === void 0 ? void 0 : node.type) === REQUEST && resultCacheKey != null && (node === null || node === void 0 ? void 0 : node.result) != null) {
|
|
1066
|
+
queue.add(() => serialiseAndSet(resultCacheKey, node.result));
|
|
1067
|
+
|
|
1068
|
+
// eslint-disable-next-line no-unused-vars
|
|
1069
|
+
let {
|
|
1070
|
+
result: _,
|
|
1071
|
+
...newNode
|
|
1072
|
+
} = node;
|
|
1073
|
+
cacheableNodes[i] = newNode;
|
|
1074
|
+
} else {
|
|
1075
|
+
cacheableNodes[i] = node;
|
|
1076
|
+
}
|
|
1034
1077
|
}
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
}
|
|
1078
|
+
let nodeCountsPerBlob = [];
|
|
1079
|
+
for (let i = 0; i * this.graph.nodesPerBlob < cacheableNodes.length; i += 1) {
|
|
1080
|
+
let nodesStartIndex = i * this.graph.nodesPerBlob;
|
|
1081
|
+
let nodesEndIndex = Math.min((i + 1) * this.graph.nodesPerBlob, cacheableNodes.length);
|
|
1082
|
+
nodeCountsPerBlob.push(nodesEndIndex - nodesStartIndex);
|
|
1083
|
+
if (!this.graph.hasCachedRequestChunk(i)) {
|
|
1084
|
+
// We assume the request graph nodes are immutable and won't change
|
|
1085
|
+
let nodesToCache = cacheableNodes.slice(nodesStartIndex, nodesEndIndex);
|
|
1086
|
+
queue.add(() => serialiseAndSet(getRequestGraphNodeKey(i, cacheKey), nodesToCache).then(() => {
|
|
1087
|
+
// Succeeded in writing to disk, save that we have completed this chunk
|
|
1088
|
+
this.graph.setCachedRequestChunk(i);
|
|
1089
|
+
}));
|
|
1090
|
+
}
|
|
1048
1091
|
}
|
|
1049
|
-
}
|
|
1050
|
-
try {
|
|
1051
1092
|
await queue.run();
|
|
1052
1093
|
|
|
1053
1094
|
// Set the request graph after the queue is flushed to avoid writing an invalid state
|
|
@@ -1056,7 +1097,7 @@ class RequestTracker {
|
|
|
1056
1097
|
nodeCountsPerBlob,
|
|
1057
1098
|
nodes: undefined
|
|
1058
1099
|
});
|
|
1059
|
-
await runCacheImprovements(() => serialiseAndSet(
|
|
1100
|
+
await runCacheImprovements(() => serialiseAndSet(`${cacheKey}/cache_metadata`, {
|
|
1060
1101
|
version: _constants.ATLASPACK_VERSION,
|
|
1061
1102
|
entries: this.options.entries,
|
|
1062
1103
|
mode: this.options.mode,
|
|
@@ -1064,15 +1105,16 @@ class RequestTracker {
|
|
|
1064
1105
|
watchBackend: this.options.watchBackend
|
|
1065
1106
|
}), () => Promise.resolve());
|
|
1066
1107
|
let opts = getWatcherOptions(this.options);
|
|
1067
|
-
let snapshotPath = _path2().default.join(this.options.cacheDir,
|
|
1108
|
+
let snapshotPath = _path2().default.join(this.options.cacheDir, snapshotKey + '.txt');
|
|
1068
1109
|
await this.options.outputFS.writeSnapshot(this.options.watchDir, snapshotPath, opts);
|
|
1069
1110
|
} catch (err) {
|
|
1070
1111
|
// If we have aborted, ignore the error and continue
|
|
1071
1112
|
if (!(signal !== null && signal !== void 0 && signal.aborted)) throw err;
|
|
1113
|
+
} finally {
|
|
1114
|
+
await runCacheImprovements(async cache => {
|
|
1115
|
+
await cache.getNativeRef().commitWriteTransaction();
|
|
1116
|
+
}, () => Promise.resolve());
|
|
1072
1117
|
}
|
|
1073
|
-
await runCacheImprovements(async cache => {
|
|
1074
|
-
await cache.getNativeRef().commitWriteTransaction();
|
|
1075
|
-
}, () => Promise.resolve());
|
|
1076
1118
|
(0, _ReporterRunner.report)({
|
|
1077
1119
|
type: 'cache',
|
|
1078
1120
|
phase: 'end',
|
|
@@ -1109,17 +1151,30 @@ function getWatcherOptions({
|
|
|
1109
1151
|
};
|
|
1110
1152
|
}
|
|
1111
1153
|
function getCacheKey(options) {
|
|
1154
|
+
if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
|
1155
|
+
const hash = (0, _rust().hashString)(`${_constants.ATLASPACK_VERSION}:${JSON.stringify(options.entries)}:${options.mode}:${options.shouldBuildLazily ? 'lazy' : 'eager'}:${options.watchBackend ?? ''}`);
|
|
1156
|
+
return `RequestTracker/${_constants.ATLASPACK_VERSION}/${hash}`;
|
|
1157
|
+
}
|
|
1112
1158
|
return (0, _rust().hashString)(`${_constants.ATLASPACK_VERSION}:${JSON.stringify(options.entries)}:${options.mode}:${options.shouldBuildLazily ? 'lazy' : 'eager'}:${options.watchBackend ?? ''}`);
|
|
1113
1159
|
}
|
|
1114
1160
|
function getRequestGraphNodeKey(index, cacheKey) {
|
|
1161
|
+
if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
|
1162
|
+
return `${cacheKey}/RequestGraph/nodes/${index}`;
|
|
1163
|
+
}
|
|
1115
1164
|
return `requestGraph-nodes-${index}-${cacheKey}`;
|
|
1116
1165
|
}
|
|
1117
1166
|
async function readAndDeserializeRequestGraph(cache, requestGraphKey, cacheKey) {
|
|
1118
1167
|
let bufferLength = 0;
|
|
1119
1168
|
const getAndDeserialize = async key => {
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1169
|
+
if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
|
1170
|
+
const buffer = await cache.getBlob(key);
|
|
1171
|
+
bufferLength += Buffer.byteLength(buffer);
|
|
1172
|
+
return (0, _buildCache().deserialize)(buffer);
|
|
1173
|
+
} else {
|
|
1174
|
+
const buffer = await cache.getLargeBlob(key);
|
|
1175
|
+
bufferLength += Buffer.byteLength(buffer);
|
|
1176
|
+
return (0, _buildCache().deserialize)(buffer);
|
|
1177
|
+
}
|
|
1123
1178
|
};
|
|
1124
1179
|
let serializedRequestGraph = await getAndDeserialize(requestGraphKey);
|
|
1125
1180
|
let nodePromises = serializedRequestGraph.nodeCountsPerBlob.map(async (nodesCount, i) => {
|
|
@@ -1141,9 +1196,9 @@ async function loadRequestGraph(options) {
|
|
|
1141
1196
|
return new RequestGraph();
|
|
1142
1197
|
}
|
|
1143
1198
|
let cacheKey = getCacheKey(options);
|
|
1144
|
-
let requestGraphKey = `requestGraph-${cacheKey}`;
|
|
1199
|
+
let requestGraphKey = (0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements') ? `${cacheKey}/RequestGraph` : `requestGraph-${cacheKey}`;
|
|
1145
1200
|
let timeout;
|
|
1146
|
-
const snapshotKey = `snapshot-${cacheKey}`;
|
|
1201
|
+
const snapshotKey = (0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements') ? `${cacheKey}/snapshot` : `snapshot-${cacheKey}`;
|
|
1147
1202
|
const snapshotPath = _path2().default.join(options.cacheDir, snapshotKey + '.txt');
|
|
1148
1203
|
const commonMeta = {
|
|
1149
1204
|
cacheKey,
|
|
@@ -1163,7 +1218,11 @@ async function loadRequestGraph(options) {
|
|
|
1163
1218
|
...commonMeta
|
|
1164
1219
|
}
|
|
1165
1220
|
});
|
|
1166
|
-
if (
|
|
1221
|
+
if ((0, _featureFlags().getFeatureFlag)('environmentDeduplication')) {
|
|
1222
|
+
await (0, _EnvironmentManager.loadEnvironmentsFromCache)(options.cache);
|
|
1223
|
+
}
|
|
1224
|
+
const hasRequestGraphInCache = (0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements') ? await options.cache.has(requestGraphKey) : await options.cache.hasLargeBlob(requestGraphKey);
|
|
1225
|
+
if (hasRequestGraphInCache) {
|
|
1167
1226
|
try {
|
|
1168
1227
|
let {
|
|
1169
1228
|
requestGraph
|
|
@@ -1230,6 +1289,20 @@ async function loadRequestGraph(options) {
|
|
|
1230
1289
|
* A wrapper around an invalidation type / method
|
|
1231
1290
|
*/
|
|
1232
1291
|
|
|
1292
|
+
/**
|
|
1293
|
+
* Details about an invalidation.
|
|
1294
|
+
*
|
|
1295
|
+
* If this is a fs events invalidation, this key will contain statistics about invalidations
|
|
1296
|
+
* by path.
|
|
1297
|
+
*
|
|
1298
|
+
* If this is a env or option invalidation, this key will contain the list of changed environment
|
|
1299
|
+
* variables or options.
|
|
1300
|
+
*/
|
|
1301
|
+
|
|
1302
|
+
/**
|
|
1303
|
+
* Number of invalidations for a given file-system event.
|
|
1304
|
+
*/
|
|
1305
|
+
|
|
1233
1306
|
/**
|
|
1234
1307
|
* Information about a certain cache invalidation type.
|
|
1235
1308
|
*/
|
|
@@ -1255,9 +1328,7 @@ async function invalidateRequestGraph(requestGraph, options, events) {
|
|
|
1255
1328
|
fn: () => requestGraph.invalidateOptionNodes(options)
|
|
1256
1329
|
}, {
|
|
1257
1330
|
key: 'fsEvents',
|
|
1258
|
-
fn:
|
|
1259
|
-
await requestGraph.respondToFSEvents(options.unstableFileInvalidations || events, options, 10000, true);
|
|
1260
|
-
}
|
|
1331
|
+
fn: () => invalidateRequestGraphFSEvents(requestGraph, options, events)
|
|
1261
1332
|
}];
|
|
1262
1333
|
const invalidations = [];
|
|
1263
1334
|
for (const invalidation of invalidationFns) {
|
|
@@ -1275,18 +1346,34 @@ async function invalidateRequestGraph(requestGraph, options, events) {
|
|
|
1275
1346
|
requestInvalidationRatio: invalidatedCount / requestCount
|
|
1276
1347
|
};
|
|
1277
1348
|
}
|
|
1278
|
-
|
|
1349
|
+
/**
|
|
1350
|
+
* Invalidate the request graph based on file-system events.
|
|
1351
|
+
*
|
|
1352
|
+
* Returns statistics about the invalidations.
|
|
1353
|
+
*/
|
|
1354
|
+
async function invalidateRequestGraphFSEvents(requestGraph, options, events) {
|
|
1355
|
+
const {
|
|
1356
|
+
invalidationsByPath
|
|
1357
|
+
} = await requestGraph.respondToFSEvents(options.unstableFileInvalidations || events, options, 10000, true);
|
|
1358
|
+
const biggestInvalidations = getBiggestFSEventsInvalidations(invalidationsByPath);
|
|
1359
|
+
return {
|
|
1360
|
+
biggestInvalidations
|
|
1361
|
+
};
|
|
1362
|
+
}
|
|
1279
1363
|
/**
|
|
1280
1364
|
* Runs an invalidation function and reports metrics.
|
|
1281
1365
|
*/
|
|
1282
1366
|
async function runInvalidation(requestGraph, invalidationFn) {
|
|
1283
|
-
const
|
|
1367
|
+
const start = _perf_hooks().performance.now();
|
|
1368
|
+
const startInvalidationCount = requestGraph.getInvalidNodeCount();
|
|
1284
1369
|
const result = await invalidationFn.fn();
|
|
1285
|
-
const count = requestGraph.
|
|
1370
|
+
const count = requestGraph.getInvalidNodeCount() - startInvalidationCount;
|
|
1371
|
+
const duration = _perf_hooks().performance.now() - start;
|
|
1286
1372
|
return {
|
|
1287
1373
|
key: invalidationFn.key,
|
|
1288
1374
|
count,
|
|
1289
|
-
|
|
1375
|
+
detail: result ?? null,
|
|
1376
|
+
duration
|
|
1290
1377
|
};
|
|
1291
1378
|
}
|
|
1292
1379
|
function logErrorOnBailout(options, snapshotPath, e) {
|
|
@@ -1328,4 +1415,19 @@ function cleanUpOrphans(graph) {
|
|
|
1328
1415
|
}
|
|
1329
1416
|
});
|
|
1330
1417
|
return removedNodeIds;
|
|
1418
|
+
}
|
|
1419
|
+
|
|
1420
|
+
/**
|
|
1421
|
+
* Returns paths that invalidated the most nodes
|
|
1422
|
+
*/
|
|
1423
|
+
function getBiggestFSEventsInvalidations(invalidationsByPath, limit = 10) {
|
|
1424
|
+
const invalidations = [];
|
|
1425
|
+
for (const [path, count] of invalidationsByPath) {
|
|
1426
|
+
invalidations.push({
|
|
1427
|
+
path,
|
|
1428
|
+
count
|
|
1429
|
+
});
|
|
1430
|
+
}
|
|
1431
|
+
invalidations.sort((a, b) => b.count - a.count);
|
|
1432
|
+
return invalidations.slice(0, limit);
|
|
1331
1433
|
}
|
package/lib/UncommittedAsset.js
CHANGED
|
@@ -53,6 +53,14 @@ var _assetUtils = require("./assetUtils");
|
|
|
53
53
|
var _types = require("./types");
|
|
54
54
|
var _utils2 = require("./utils");
|
|
55
55
|
var _projectPath = require("./projectPath");
|
|
56
|
+
function _featureFlags() {
|
|
57
|
+
const data = require("@atlaspack/feature-flags");
|
|
58
|
+
_featureFlags = function () {
|
|
59
|
+
return data;
|
|
60
|
+
};
|
|
61
|
+
return data;
|
|
62
|
+
}
|
|
63
|
+
var _EnvironmentManager = require("./EnvironmentManager");
|
|
56
64
|
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
|
57
65
|
class UncommittedAsset {
|
|
58
66
|
constructor({
|
|
@@ -133,6 +141,8 @@ class UncommittedAsset {
|
|
|
133
141
|
hash = (0, _rust().hashBuffer)(content);
|
|
134
142
|
size = content.length;
|
|
135
143
|
}
|
|
144
|
+
|
|
145
|
+
// Maybe we should just store this in a file instead of LMDB
|
|
136
146
|
await this.options.cache.setBlob(contentKey, content);
|
|
137
147
|
return {
|
|
138
148
|
size,
|
|
@@ -184,6 +194,10 @@ class UncommittedAsset {
|
|
|
184
194
|
this.content = buffer;
|
|
185
195
|
this.clearAST();
|
|
186
196
|
}
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* @deprecated This has been broken on any cache other than FSCache for a long time.
|
|
200
|
+
*/
|
|
187
201
|
setStream(stream) {
|
|
188
202
|
this.content = stream;
|
|
189
203
|
this.clearAST();
|
|
@@ -248,6 +262,10 @@ class UncommittedAsset {
|
|
|
248
262
|
this.value.astGenerator = null;
|
|
249
263
|
}
|
|
250
264
|
getCacheKey(key) {
|
|
265
|
+
if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
|
266
|
+
const filePath = (0, _projectPath.fromProjectPathRelative)(this.value.filePath);
|
|
267
|
+
return `Asset/${_constants.ATLASPACK_VERSION}/${filePath}/${this.value.id}/${key}`;
|
|
268
|
+
}
|
|
251
269
|
return (0, _rust().hashString)(_constants.ATLASPACK_VERSION + key + this.value.id);
|
|
252
270
|
}
|
|
253
271
|
addDependency(opts) {
|
|
@@ -261,7 +279,7 @@ class UncommittedAsset {
|
|
|
261
279
|
...rest,
|
|
262
280
|
// $FlowFixMe "convert" the $ReadOnlyMaps to the interal mutable one
|
|
263
281
|
symbols,
|
|
264
|
-
env: (0, _Environment.mergeEnvironments)(this.options.projectRoot, this.value.env, env),
|
|
282
|
+
env: (0, _Environment.mergeEnvironments)(this.options.projectRoot, (0, _EnvironmentManager.fromEnvironmentId)(this.value.env), env),
|
|
265
283
|
sourceAssetId: this.value.id,
|
|
266
284
|
sourcePath: (0, _projectPath.fromProjectPath)(this.options.projectRoot, this.value.filePath)
|
|
267
285
|
});
|
|
@@ -301,7 +319,7 @@ class UncommittedAsset {
|
|
|
301
319
|
bundleBehavior: result.bundleBehavior ?? (this.value.bundleBehavior == null ? null : _types.BundleBehaviorNames[this.value.bundleBehavior]),
|
|
302
320
|
isBundleSplittable: result.isBundleSplittable ?? this.value.isBundleSplittable,
|
|
303
321
|
isSource: this.value.isSource,
|
|
304
|
-
env: (0, _Environment.mergeEnvironments)(this.options.projectRoot, this.value.env, result.env),
|
|
322
|
+
env: (0, _Environment.mergeEnvironments)(this.options.projectRoot, (0, _EnvironmentManager.fromEnvironmentId)(this.value.env), result.env),
|
|
305
323
|
dependencies: this.value.type === result.type ? new Map(this.value.dependencies) : new Map(),
|
|
306
324
|
meta: {
|
|
307
325
|
...this.value.meta,
|
package/lib/applyRuntimes.js
CHANGED
|
@@ -69,6 +69,7 @@ function _utils() {
|
|
|
69
69
|
};
|
|
70
70
|
return data;
|
|
71
71
|
}
|
|
72
|
+
var _EnvironmentManager = require("./EnvironmentManager");
|
|
72
73
|
function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
|
|
73
74
|
function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
|
|
74
75
|
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
|
@@ -164,7 +165,7 @@ async function applyRuntimes({
|
|
|
164
165
|
let assetGroup = {
|
|
165
166
|
code,
|
|
166
167
|
filePath: (0, _projectPath.toProjectPath)(options.projectRoot, sourceName),
|
|
167
|
-
env: (0, _Environment.mergeEnvironments)(options.projectRoot, bundle.env, env),
|
|
168
|
+
env: (0, _Environment.mergeEnvironments)(options.projectRoot, (0, _EnvironmentManager.fromEnvironmentId)(bundle.env), env),
|
|
168
169
|
// Runtime assets should be considered source, as they should be
|
|
169
170
|
// e.g. compiled to run in the target environment
|
|
170
171
|
isSource: true
|
package/lib/assetUtils.js
CHANGED
|
@@ -66,10 +66,11 @@ function _profiler() {
|
|
|
66
66
|
return data;
|
|
67
67
|
}
|
|
68
68
|
var _IdentifierRegistry = require("./IdentifierRegistry");
|
|
69
|
+
var _EnvironmentManager = require("./EnvironmentManager");
|
|
69
70
|
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
|
70
71
|
function createAssetIdFromOptions(options) {
|
|
71
72
|
const data = {
|
|
72
|
-
environmentId: options.env
|
|
73
|
+
environmentId: (0, _EnvironmentManager.toEnvironmentId)(options.env),
|
|
73
74
|
filePath: options.filePath,
|
|
74
75
|
code: options.code,
|
|
75
76
|
pipeline: options.pipeline,
|
package/lib/public/Asset.js
CHANGED
|
@@ -23,6 +23,7 @@ var _Environment2 = require("../Environment");
|
|
|
23
23
|
var _projectPath = require("../projectPath");
|
|
24
24
|
var _types = require("../types");
|
|
25
25
|
var _utils = require("../utils");
|
|
26
|
+
var _EnvironmentManager = require("../EnvironmentManager");
|
|
26
27
|
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
|
27
28
|
const inspect = Symbol.for('nodejs.util.inspect.custom');
|
|
28
29
|
const uncommittedAssetValueToAsset = new WeakMap();
|
|
@@ -62,7 +63,7 @@ class BaseAsset {
|
|
|
62
63
|
return this.#asset.value.type;
|
|
63
64
|
}
|
|
64
65
|
get env() {
|
|
65
|
-
return new _Environment.default(this.#asset.value.env, this.#asset.options);
|
|
66
|
+
return new _Environment.default((0, _EnvironmentManager.fromEnvironmentId)(this.#asset.value.env), this.#asset.options);
|
|
66
67
|
}
|
|
67
68
|
get fs() {
|
|
68
69
|
return this.#asset.options.inputFS;
|
|
@@ -142,7 +143,7 @@ class Asset extends BaseAsset {
|
|
|
142
143
|
return this;
|
|
143
144
|
}
|
|
144
145
|
get env() {
|
|
145
|
-
this.#env ??= new _Environment.default(this.#asset.value.env, this.#asset.options);
|
|
146
|
+
this.#env ??= new _Environment.default((0, _EnvironmentManager.fromEnvironmentId)(this.#asset.value.env), this.#asset.options);
|
|
146
147
|
return this.#env;
|
|
147
148
|
}
|
|
148
149
|
get stats() {
|
package/lib/public/Bundle.js
CHANGED
|
@@ -40,6 +40,7 @@ var _Dependency = require("./Dependency");
|
|
|
40
40
|
var _Target = _interopRequireDefault(require("./Target"));
|
|
41
41
|
var _types = require("../types");
|
|
42
42
|
var _projectPath = require("../projectPath");
|
|
43
|
+
var _EnvironmentManager = require("../EnvironmentManager");
|
|
43
44
|
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
|
44
45
|
const inspect = Symbol.for('nodejs.util.inspect.custom');
|
|
45
46
|
const internalBundleToBundle = new (_utils().DefaultWeakMap)(() => new (_utils().DefaultWeakMap)(() => new WeakMap()));
|
|
@@ -99,7 +100,7 @@ class Bundle {
|
|
|
99
100
|
return this.#bundle.type;
|
|
100
101
|
}
|
|
101
102
|
get env() {
|
|
102
|
-
return new _Environment.default(this.#bundle.env, this.#options);
|
|
103
|
+
return new _Environment.default((0, _EnvironmentManager.fromEnvironmentId)(this.#bundle.env), this.#options);
|
|
103
104
|
}
|
|
104
105
|
get needsStableName() {
|
|
105
106
|
return this.#bundle.needsStableName;
|
|
@@ -32,6 +32,13 @@ var _Dependency = _interopRequireWildcard(require("./Dependency"));
|
|
|
32
32
|
var _Target = require("./Target");
|
|
33
33
|
var _utils = require("../utils");
|
|
34
34
|
var _BundleGroup = _interopRequireWildcard(require("./BundleGroup"));
|
|
35
|
+
function _featureFlags() {
|
|
36
|
+
const data = require("@atlaspack/feature-flags");
|
|
37
|
+
_featureFlags = function () {
|
|
38
|
+
return data;
|
|
39
|
+
};
|
|
40
|
+
return data;
|
|
41
|
+
}
|
|
35
42
|
function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
|
|
36
43
|
function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
|
|
37
44
|
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
|
@@ -263,11 +270,20 @@ class BundleGraph {
|
|
|
263
270
|
}
|
|
264
271
|
for (let bundle of bundles) {
|
|
265
272
|
const conditions = bundleConditions.get(bundle.id) ?? new Map();
|
|
266
|
-
conditions.
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
273
|
+
const currentCondition = conditions.get(cond.key);
|
|
274
|
+
if ((0, _featureFlags().getFeatureFlag)('conditionalBundlingReporterSameConditionFix')) {
|
|
275
|
+
conditions.set(cond.key, {
|
|
276
|
+
bundle,
|
|
277
|
+
ifTrueBundles: [...((currentCondition === null || currentCondition === void 0 ? void 0 : currentCondition.ifTrueBundles) ?? []), ...ifTrueBundles],
|
|
278
|
+
ifFalseBundles: [...((currentCondition === null || currentCondition === void 0 ? void 0 : currentCondition.ifFalseBundles) ?? []), ...ifFalseBundles]
|
|
279
|
+
});
|
|
280
|
+
} else {
|
|
281
|
+
conditions.set(cond.key, {
|
|
282
|
+
bundle,
|
|
283
|
+
ifTrueBundles,
|
|
284
|
+
ifFalseBundles
|
|
285
|
+
});
|
|
286
|
+
}
|
|
271
287
|
bundleConditions.set(bundle.id, conditions);
|
|
272
288
|
}
|
|
273
289
|
}
|