@atlaspack/cache 2.13.3-dev.144 → 2.13.3-dev.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +109 -0
- package/index.d.ts +0 -4
- package/lib/FSCache.js +21 -0
- package/lib/LMDBLiteCache.js +155 -39
- package/lib/index.js +0 -11
- package/package.json +10 -10
- package/src/FSCache.js +9 -0
- package/src/LMDBLiteCache.js +141 -35
- package/src/index.js +0 -1
- package/test/LMDBLiteCache.test.js +214 -4
- package/test/workerThreadsTest.js +42 -0
- package/lib/LMDBCache.js +0 -136
- package/src/LMDBCache.js +0 -139
package/CHANGELOG.md
CHANGED
@@ -1,5 +1,114 @@
|
|
1
1
|
# @atlaspack/cache
|
2
2
|
|
3
|
+
## 3.2.5
|
4
|
+
|
5
|
+
### Patch Changes
|
6
|
+
|
7
|
+
- Updated dependencies [[`0999fb7`](https://github.com/atlassian-labs/atlaspack/commit/0999fb78da519a6c7582d212883e515fcf6c1252), [`35fdd4b`](https://github.com/atlassian-labs/atlaspack/commit/35fdd4b52da0af20f74667f7b8adfb2f90279b7c), [`6dd4ccb`](https://github.com/atlassian-labs/atlaspack/commit/6dd4ccb753541de32322d881f973d571dd57e4ca)]:
|
8
|
+
- @atlaspack/fs@2.15.5
|
9
|
+
- @atlaspack/rust@3.3.5
|
10
|
+
- @atlaspack/logger@2.14.10
|
11
|
+
- @atlaspack/utils@2.14.10
|
12
|
+
|
13
|
+
## 3.2.4
|
14
|
+
|
15
|
+
### Patch Changes
|
16
|
+
|
17
|
+
- [#583](https://github.com/atlassian-labs/atlaspack/pull/583) [`124b7ff`](https://github.com/atlassian-labs/atlaspack/commit/124b7fff44f71aac9fbad289a9a9509b3dfc9aaa) Thanks [@yamadapc](https://github.com/yamadapc)! - Fix problem where cache writes could start to fail during a V3 build
|
18
|
+
|
19
|
+
- Updated dependencies [[`124b7ff`](https://github.com/atlassian-labs/atlaspack/commit/124b7fff44f71aac9fbad289a9a9509b3dfc9aaa), [`e052521`](https://github.com/atlassian-labs/atlaspack/commit/e0525210850ed1606146eb86991049cf567c5dec), [`15c6d70`](https://github.com/atlassian-labs/atlaspack/commit/15c6d7000bd89da876bc590aa75b17a619a41896), [`e4d966c`](https://github.com/atlassian-labs/atlaspack/commit/e4d966c3c9c4292c5013372ae65b10d19d4bacc6), [`209692f`](https://github.com/atlassian-labs/atlaspack/commit/209692ffb11eae103a0d65c5e1118a5aa1625818), [`42a775d`](https://github.com/atlassian-labs/atlaspack/commit/42a775de8eec638ad188f3271964170d8c04d84b), [`29c2f10`](https://github.com/atlassian-labs/atlaspack/commit/29c2f106de9679adfb5afa04e1910471dc65a427), [`f4da1e1`](https://github.com/atlassian-labs/atlaspack/commit/f4da1e120e73eeb5e8b8927f05e88f04d6148c7b), [`1ef91fc`](https://github.com/atlassian-labs/atlaspack/commit/1ef91fcc863fdd2831511937083dbbc1263b3d9d)]:
|
20
|
+
- @atlaspack/rust@3.3.4
|
21
|
+
- @atlaspack/fs@2.15.4
|
22
|
+
- @atlaspack/feature-flags@2.16.0
|
23
|
+
- @atlaspack/logger@2.14.9
|
24
|
+
- @atlaspack/utils@2.14.9
|
25
|
+
|
26
|
+
## 3.2.3
|
27
|
+
|
28
|
+
### Patch Changes
|
29
|
+
|
30
|
+
- Updated dependencies [[`30f6017`](https://github.com/atlassian-labs/atlaspack/commit/30f60175ba4d272c5fc193973c63bc298584775b), [`3a3e8e7`](https://github.com/atlassian-labs/atlaspack/commit/3a3e8e7be9e2dffd7304436d792f0f595d59665a), [`1ab0a27`](https://github.com/atlassian-labs/atlaspack/commit/1ab0a275aeca40350415e2b03e7440d1dddc6228), [`b8a4ae8`](https://github.com/atlassian-labs/atlaspack/commit/b8a4ae8f83dc0a83d8b145c5f729936ce52080a3)]:
|
31
|
+
- @atlaspack/feature-flags@2.15.1
|
32
|
+
- @atlaspack/fs@2.15.3
|
33
|
+
- @atlaspack/rust@3.3.3
|
34
|
+
- @atlaspack/utils@2.14.8
|
35
|
+
- @atlaspack/logger@2.14.8
|
36
|
+
|
37
|
+
## 3.2.2
|
38
|
+
|
39
|
+
### Patch Changes
|
40
|
+
|
41
|
+
- Updated dependencies [[`a1773d2`](https://github.com/atlassian-labs/atlaspack/commit/a1773d2a62d0ef7805ac7524621dcabcc1afe929), [`556d6ab`](https://github.com/atlassian-labs/atlaspack/commit/556d6ab8ede759fa7f37fcd3f4da336ef1c55e8f)]:
|
42
|
+
- @atlaspack/feature-flags@2.15.0
|
43
|
+
- @atlaspack/logger@2.14.7
|
44
|
+
- @atlaspack/rust@3.3.2
|
45
|
+
- @atlaspack/fs@2.15.2
|
46
|
+
- @atlaspack/utils@2.14.7
|
47
|
+
|
48
|
+
## 3.2.1
|
49
|
+
|
50
|
+
### Patch Changes
|
51
|
+
|
52
|
+
- Updated dependencies [[`e0f5337`](https://github.com/atlassian-labs/atlaspack/commit/e0f533757bd1019dbd108a04952c87da15286e09)]:
|
53
|
+
- @atlaspack/feature-flags@2.14.4
|
54
|
+
- @atlaspack/rust@3.3.1
|
55
|
+
- @atlaspack/fs@2.15.1
|
56
|
+
- @atlaspack/utils@2.14.6
|
57
|
+
- @atlaspack/logger@2.14.6
|
58
|
+
|
59
|
+
## 3.2.0
|
60
|
+
|
61
|
+
### Minor Changes
|
62
|
+
|
63
|
+
- [#531](https://github.com/atlassian-labs/atlaspack/pull/531) [`d2c50c2`](https://github.com/atlassian-labs/atlaspack/commit/d2c50c2c020888b33bb25b8690d9320c2b69e2a6) Thanks [@yamadapc](https://github.com/yamadapc)! - Add way to iterate LMDB cache keys
|
64
|
+
|
65
|
+
### Patch Changes
|
66
|
+
|
67
|
+
- Updated dependencies [[`11d6f16`](https://github.com/atlassian-labs/atlaspack/commit/11d6f16b6397dee2f217167e5c98b39edb63f7a7), [`e2ba0f6`](https://github.com/atlassian-labs/atlaspack/commit/e2ba0f69702656f3d1ce95ab1454e35062b13b39), [`d2c50c2`](https://github.com/atlassian-labs/atlaspack/commit/d2c50c2c020888b33bb25b8690d9320c2b69e2a6), [`46a90dc`](https://github.com/atlassian-labs/atlaspack/commit/46a90dccd019a26b222c878a92d23acc75dc67c5), [`4c17141`](https://github.com/atlassian-labs/atlaspack/commit/4c1714103dab2aa9039c488f381551d2b65d1d01)]:
|
68
|
+
- @atlaspack/feature-flags@2.14.3
|
69
|
+
- @atlaspack/rust@3.3.0
|
70
|
+
- @atlaspack/fs@2.15.0
|
71
|
+
- @atlaspack/utils@2.14.5
|
72
|
+
- @atlaspack/logger@2.14.5
|
73
|
+
|
74
|
+
## 3.1.0
|
75
|
+
|
76
|
+
### Minor Changes
|
77
|
+
|
78
|
+
- [#525](https://github.com/atlassian-labs/atlaspack/pull/525) [`cb9da16`](https://github.com/atlassian-labs/atlaspack/commit/cb9da16fb2648e7f53c64df0313f60d5fb8970cc) Thanks [@yamadapc](https://github.com/yamadapc)! - Fix issues with large blob cache writes, run cache writes in a write transaction
|
79
|
+
|
80
|
+
### Patch Changes
|
81
|
+
|
82
|
+
- Updated dependencies [[`1a2c14c`](https://github.com/atlassian-labs/atlaspack/commit/1a2c14c3cd4587551cc12e94d0680c8b71ea12bf), [`cb9da16`](https://github.com/atlassian-labs/atlaspack/commit/cb9da16fb2648e7f53c64df0313f60d5fb8970cc)]:
|
83
|
+
- @atlaspack/rust@3.2.0
|
84
|
+
- @atlaspack/fs@2.14.4
|
85
|
+
- @atlaspack/logger@2.14.4
|
86
|
+
- @atlaspack/utils@2.14.4
|
87
|
+
|
88
|
+
## 3.0.1
|
89
|
+
|
90
|
+
### Patch Changes
|
91
|
+
|
92
|
+
- Updated dependencies [[`f27d39e`](https://github.com/atlassian-labs/atlaspack/commit/f27d39e767b06def059944b3bc5fd50797eaea96)]:
|
93
|
+
- @atlaspack/rust@3.1.1
|
94
|
+
- @atlaspack/fs@2.14.3
|
95
|
+
- @atlaspack/logger@2.14.3
|
96
|
+
- @atlaspack/utils@2.14.3
|
97
|
+
|
98
|
+
## 3.0.0
|
99
|
+
|
100
|
+
### Major Changes
|
101
|
+
|
102
|
+
- [#512](https://github.com/atlassian-labs/atlaspack/pull/512) [`8f4e6c1`](https://github.com/atlassian-labs/atlaspack/commit/8f4e6c1b0e7c1fd48624afda48c1dcc599f1460f) Thanks [@yamadapc](https://github.com/yamadapc)! - Remove LMDB cache back-end
|
103
|
+
|
104
|
+
### Patch Changes
|
105
|
+
|
106
|
+
- Updated dependencies [[`a891d65`](https://github.com/atlassian-labs/atlaspack/commit/a891d652bc4eb3d757d381adf65c5083f706effc), [`d02eab9`](https://github.com/atlassian-labs/atlaspack/commit/d02eab95eb60bf7457e0869af0b773608592c0e6), [`fb87a90`](https://github.com/atlassian-labs/atlaspack/commit/fb87a901973776b33ca4ce530e9d71669a9bd36d), [`7b9e8cf`](https://github.com/atlassian-labs/atlaspack/commit/7b9e8cf29e01a98e72e46b2b2fb74ccc514f4463)]:
|
107
|
+
- @atlaspack/rust@3.1.0
|
108
|
+
- @atlaspack/fs@2.14.2
|
109
|
+
- @atlaspack/utils@2.14.2
|
110
|
+
- @atlaspack/logger@2.14.2
|
111
|
+
|
3
112
|
## 2.13.3
|
4
113
|
|
5
114
|
### Patch Changes
|
package/index.d.ts
CHANGED
package/lib/FSCache.js
CHANGED
@@ -25,6 +25,20 @@ function _util() {
|
|
25
25
|
};
|
26
26
|
return data;
|
27
27
|
}
|
28
|
+
function _rust() {
|
29
|
+
const data = require("@atlaspack/rust");
|
30
|
+
_rust = function () {
|
31
|
+
return data;
|
32
|
+
};
|
33
|
+
return data;
|
34
|
+
}
|
35
|
+
function _featureFlags() {
|
36
|
+
const data = require("@atlaspack/feature-flags");
|
37
|
+
_featureFlags = function () {
|
38
|
+
return data;
|
39
|
+
};
|
40
|
+
return data;
|
41
|
+
}
|
28
42
|
function _logger() {
|
29
43
|
const data = _interopRequireDefault(require("@atlaspack/logger"));
|
30
44
|
_logger = function () {
|
@@ -62,6 +76,10 @@ class FSCache {
|
|
62
76
|
await Promise.all(dirPromises);
|
63
77
|
}
|
64
78
|
_getCachePath(cacheId) {
|
79
|
+
if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
80
|
+
const cleanId = (0, _rust().hashString)(cacheId);
|
81
|
+
return _path().default.join(this.dir, cleanId.slice(0, 2), cleanId.slice(2));
|
82
|
+
}
|
65
83
|
return _path().default.join(this.dir, cacheId.slice(0, 2), cacheId.slice(2));
|
66
84
|
}
|
67
85
|
getStream(key) {
|
@@ -91,6 +109,9 @@ class FSCache {
|
|
91
109
|
}
|
92
110
|
}
|
93
111
|
#getFilePath(key, index) {
|
112
|
+
if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
113
|
+
return _path().default.join(this.dir, `${(0, _rust().hashString)(key)}-${index}`);
|
114
|
+
}
|
94
115
|
return _path().default.join(this.dir, `${key}-${index}`);
|
95
116
|
}
|
96
117
|
async #unlinkChunks(key, index) {
|
package/lib/LMDBLiteCache.js
CHANGED
@@ -12,6 +12,13 @@ function _buildCache() {
|
|
12
12
|
};
|
13
13
|
return data;
|
14
14
|
}
|
15
|
+
function _featureFlags() {
|
16
|
+
const data = require("@atlaspack/feature-flags");
|
17
|
+
_featureFlags = function () {
|
18
|
+
return data;
|
19
|
+
};
|
20
|
+
return data;
|
21
|
+
}
|
15
22
|
function _rust() {
|
16
23
|
const data = require("@atlaspack/rust");
|
17
24
|
_rust = function () {
|
@@ -19,16 +26,16 @@ function _rust() {
|
|
19
26
|
};
|
20
27
|
return data;
|
21
28
|
}
|
22
|
-
function
|
23
|
-
const data = _interopRequireDefault(require("
|
24
|
-
|
29
|
+
function _fs() {
|
30
|
+
const data = _interopRequireDefault(require("fs"));
|
31
|
+
_fs = function () {
|
25
32
|
return data;
|
26
33
|
};
|
27
34
|
return data;
|
28
35
|
}
|
29
|
-
function
|
30
|
-
const data = _interopRequireDefault(require("
|
31
|
-
|
36
|
+
function _ncp() {
|
37
|
+
const data = _interopRequireDefault(require("ncp"));
|
38
|
+
_ncp = function () {
|
32
39
|
return data;
|
33
40
|
};
|
34
41
|
return data;
|
@@ -40,25 +47,48 @@ function _util() {
|
|
40
47
|
};
|
41
48
|
return data;
|
42
49
|
}
|
43
|
-
function
|
50
|
+
function _stream() {
|
51
|
+
const data = _interopRequireDefault(require("stream"));
|
52
|
+
_stream = function () {
|
53
|
+
return data;
|
54
|
+
};
|
55
|
+
return data;
|
56
|
+
}
|
57
|
+
function _path() {
|
58
|
+
const data = _interopRequireDefault(require("path"));
|
59
|
+
_path = function () {
|
60
|
+
return data;
|
61
|
+
};
|
62
|
+
return data;
|
63
|
+
}
|
64
|
+
function _fs2() {
|
44
65
|
const data = require("@atlaspack/fs");
|
45
|
-
|
66
|
+
_fs2 = function () {
|
46
67
|
return data;
|
47
68
|
};
|
48
69
|
return data;
|
49
70
|
}
|
50
71
|
var _package = _interopRequireDefault(require("../package.json"));
|
51
72
|
var _FSCache = require("./FSCache");
|
73
|
+
function _logger() {
|
74
|
+
const data = require("@atlaspack/logger");
|
75
|
+
_logger = function () {
|
76
|
+
return data;
|
77
|
+
};
|
78
|
+
return data;
|
79
|
+
}
|
52
80
|
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
53
81
|
// $FlowFixMe
|
82
|
+
const ncpAsync = (0, _util().promisify)(_ncp().default);
|
54
83
|
class LmdbWrapper {
|
55
84
|
constructor(lmdb) {
|
56
85
|
this.lmdb = lmdb;
|
57
|
-
|
58
|
-
|
59
|
-
this
|
60
|
-
|
61
|
-
|
86
|
+
}
|
87
|
+
has(key) {
|
88
|
+
return this.lmdb.hasSync(key);
|
89
|
+
}
|
90
|
+
async delete(key) {
|
91
|
+
await this.lmdb.delete(key);
|
62
92
|
}
|
63
93
|
get(key) {
|
64
94
|
return this.lmdb.getSync(key);
|
@@ -67,7 +97,19 @@ class LmdbWrapper {
|
|
67
97
|
const buffer = typeof value === 'string' ? Buffer.from(value) : value;
|
68
98
|
await this.lmdb.put(key, buffer);
|
69
99
|
}
|
70
|
-
|
100
|
+
*keys() {
|
101
|
+
const PAGE_SIZE = 10000000;
|
102
|
+
let currentKeys = this.lmdb.keysSync(0, PAGE_SIZE);
|
103
|
+
while (currentKeys.length > 0) {
|
104
|
+
for (const key of currentKeys) {
|
105
|
+
yield key;
|
106
|
+
}
|
107
|
+
currentKeys = this.lmdb.keysSync(currentKeys.length, PAGE_SIZE);
|
108
|
+
}
|
109
|
+
}
|
110
|
+
compact(targetPath) {
|
111
|
+
this.lmdb.compact(targetPath);
|
112
|
+
}
|
71
113
|
}
|
72
114
|
exports.LmdbWrapper = LmdbWrapper;
|
73
115
|
function open(directory
|
@@ -81,9 +123,14 @@ function open(directory
|
|
81
123
|
}
|
82
124
|
const pipeline = (0, _util().promisify)(_stream().default.pipeline);
|
83
125
|
class LMDBLiteCache {
|
126
|
+
/**
|
127
|
+
* Directory where we store raw files.
|
128
|
+
*/
|
129
|
+
|
84
130
|
constructor(cacheDir) {
|
85
|
-
this.fs = new (
|
131
|
+
this.fs = new (_fs2().NodeFS)();
|
86
132
|
this.dir = cacheDir;
|
133
|
+
this.cacheFilesDirectory = _path().default.join(cacheDir, 'files');
|
87
134
|
this.fsCache = new _FSCache.FSCache(this.fs, cacheDir);
|
88
135
|
this.store = open(cacheDir, {
|
89
136
|
name: 'parcel-cache',
|
@@ -99,7 +146,10 @@ class LMDBLiteCache {
|
|
99
146
|
return this.store.lmdb;
|
100
147
|
}
|
101
148
|
async ensure() {
|
102
|
-
|
149
|
+
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
150
|
+
await this.fsCache.ensure();
|
151
|
+
}
|
152
|
+
await this.fs.mkdirp(this.cacheFilesDirectory);
|
103
153
|
return Promise.resolve();
|
104
154
|
}
|
105
155
|
serialize() {
|
@@ -111,7 +161,7 @@ class LMDBLiteCache {
|
|
111
161
|
return new LMDBLiteCache(cache.dir);
|
112
162
|
}
|
113
163
|
has(key) {
|
114
|
-
return Promise.resolve(this.store.
|
164
|
+
return Promise.resolve(this.store.has(key));
|
115
165
|
}
|
116
166
|
get(key) {
|
117
167
|
let data = this.store.get(key);
|
@@ -124,10 +174,20 @@ class LMDBLiteCache {
|
|
124
174
|
await this.setBlob(key, (0, _buildCache().serialize)(value));
|
125
175
|
}
|
126
176
|
getStream(key) {
|
127
|
-
|
177
|
+
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
178
|
+
return this.fs.createReadStream(_path().default.join(this.dir, key));
|
179
|
+
}
|
180
|
+
return _fs().default.createReadStream(this.getFileKey(key));
|
128
181
|
}
|
129
|
-
setStream(key, stream) {
|
130
|
-
|
182
|
+
async setStream(key, stream) {
|
183
|
+
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
184
|
+
return pipeline(stream, this.fs.createWriteStream(_path().default.join(this.dir, key)));
|
185
|
+
}
|
186
|
+
const filePath = this.getFileKey(key);
|
187
|
+
await _fs().default.promises.mkdir(_path().default.dirname(filePath), {
|
188
|
+
recursive: true
|
189
|
+
});
|
190
|
+
return pipeline(stream, _fs().default.createWriteStream(filePath));
|
131
191
|
}
|
132
192
|
|
133
193
|
// eslint-disable-next-line require-await
|
@@ -147,31 +207,87 @@ class LMDBLiteCache {
|
|
147
207
|
getBuffer(key) {
|
148
208
|
return Promise.resolve(this.store.get(key));
|
149
209
|
}
|
150
|
-
#getFilePath(key, index) {
|
151
|
-
return _path().default.join(this.dir, `${key}-${index}`);
|
152
|
-
}
|
153
210
|
hasLargeBlob(key) {
|
154
|
-
|
211
|
+
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
212
|
+
return this.fsCache.hasLargeBlob(key);
|
213
|
+
}
|
214
|
+
return _fs().default.promises.access(this.getFileKey(key), _fs().default.constants.F_OK).then(() => true).catch(() => false);
|
155
215
|
}
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
216
|
+
getLargeBlob(key) {
|
217
|
+
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
218
|
+
return this.fsCache.getLargeBlob(key);
|
219
|
+
}
|
220
|
+
return _fs().default.promises.readFile(this.getFileKey(key));
|
160
221
|
}
|
161
|
-
|
162
|
-
// eslint-disable-next-line require-await
|
163
222
|
async setLargeBlob(key, contents, options) {
|
164
|
-
|
223
|
+
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
224
|
+
return this.fsCache.setLargeBlob(key, contents, options);
|
225
|
+
}
|
226
|
+
const targetPath = this.getFileKey(key);
|
227
|
+
await _fs().default.promises.mkdir(_path().default.dirname(targetPath), {
|
228
|
+
recursive: true
|
229
|
+
});
|
230
|
+
return _fs().default.promises.writeFile(targetPath, contents);
|
165
231
|
}
|
232
|
+
|
233
|
+
/**
|
234
|
+
* @deprecated Use store.delete instead.
|
235
|
+
*/
|
166
236
|
deleteLargeBlob(key) {
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
237
|
+
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
238
|
+
return this.fsCache.deleteLargeBlob(key);
|
239
|
+
}
|
240
|
+
return this.store.delete(key);
|
241
|
+
}
|
242
|
+
keys() {
|
243
|
+
return this.store.keys();
|
244
|
+
}
|
245
|
+
async compact(targetPath) {
|
246
|
+
await _fs().default.promises.mkdir(targetPath, {
|
247
|
+
recursive: true
|
248
|
+
});
|
249
|
+
const files = await _fs().default.promises.readdir(this.dir);
|
250
|
+
// copy all files except data.mdb and lock.mdb to the target path (recursive)
|
251
|
+
for (const file of files) {
|
252
|
+
const filePath = _path().default.join(this.dir, file);
|
253
|
+
if (file === 'data.mdb' || file === 'lock.mdb') {
|
254
|
+
continue;
|
255
|
+
}
|
256
|
+
await ncpAsync(filePath, _path().default.join(targetPath, file));
|
257
|
+
}
|
258
|
+
this.store.compact(_path().default.join(targetPath, 'data.mdb'));
|
259
|
+
}
|
260
|
+
refresh() {}
|
261
|
+
|
262
|
+
/**
|
263
|
+
* Streams, packages are stored in files instead of LMDB.
|
264
|
+
*
|
265
|
+
* On this case, if a cache key happens to have a parent traversal, ../..
|
266
|
+
* it is treated specially
|
267
|
+
*
|
268
|
+
* That is, something/../something and something are meant to be different
|
269
|
+
* keys.
|
270
|
+
*
|
271
|
+
* Plus we do not want to store values outside of the cache directory.
|
272
|
+
*/
|
273
|
+
getFileKey(key) {
|
274
|
+
const cleanKey = key.split('/').map(part => {
|
275
|
+
if (part === '..') {
|
276
|
+
return '$$__parent_dir$$';
|
277
|
+
}
|
278
|
+
return part;
|
279
|
+
}).join('/');
|
280
|
+
return _path().default.join(this.cacheFilesDirectory, cleanKey);
|
281
|
+
}
|
282
|
+
async clear() {
|
283
|
+
await (0, _logger().instrumentAsync)('LMDBLiteCache::clear', async () => {
|
284
|
+
const keys = await this.keys();
|
285
|
+
for (const key of keys) {
|
286
|
+
await this.store.delete(key);
|
287
|
+
}
|
288
|
+
await this.fs.rimraf(this.cacheFilesDirectory);
|
289
|
+
await this.fs.mkdirp(this.cacheFilesDirectory);
|
290
|
+
});
|
175
291
|
}
|
176
292
|
}
|
177
293
|
exports.LMDBLiteCache = LMDBLiteCache;
|
package/lib/index.js
CHANGED
@@ -25,17 +25,6 @@ Object.keys(_IDBCache).forEach(function (key) {
|
|
25
25
|
}
|
26
26
|
});
|
27
27
|
});
|
28
|
-
var _LMDBCache = require("./LMDBCache");
|
29
|
-
Object.keys(_LMDBCache).forEach(function (key) {
|
30
|
-
if (key === "default" || key === "__esModule") return;
|
31
|
-
if (key in exports && exports[key] === _LMDBCache[key]) return;
|
32
|
-
Object.defineProperty(exports, key, {
|
33
|
-
enumerable: true,
|
34
|
-
get: function () {
|
35
|
-
return _LMDBCache[key];
|
36
|
-
}
|
37
|
-
});
|
38
|
-
});
|
39
28
|
var _LMDBLiteCache = require("./LMDBLiteCache");
|
40
29
|
Object.keys(_LMDBLiteCache).forEach(function (key) {
|
41
30
|
if (key === "default" || key === "__esModule") return;
|
package/package.json
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
{
|
2
2
|
"name": "@atlaspack/cache",
|
3
3
|
"description": "Interface for defining caches and file-system, IDB and LMDB implementations.",
|
4
|
-
"version": "2.13.3-dev.
|
4
|
+
"version": "2.13.3-dev.146+33cba81e9",
|
5
5
|
"license": "(MIT OR Apache-2.0)",
|
6
6
|
"type": "commonjs",
|
7
7
|
"publishConfig": {
|
@@ -23,19 +23,19 @@
|
|
23
23
|
"check-ts": "tsc --noEmit index.d.ts"
|
24
24
|
},
|
25
25
|
"dependencies": {
|
26
|
-
"@atlaspack/build-cache": "2.13.3-dev.
|
27
|
-
"@atlaspack/
|
28
|
-
"@atlaspack/
|
29
|
-
"@atlaspack/
|
30
|
-
"@atlaspack/
|
31
|
-
"
|
26
|
+
"@atlaspack/build-cache": "2.13.3-dev.146+33cba81e9",
|
27
|
+
"@atlaspack/feature-flags": "2.14.1-dev.146+33cba81e9",
|
28
|
+
"@atlaspack/fs": "2.14.1-dev.146+33cba81e9",
|
29
|
+
"@atlaspack/logger": "2.14.1-dev.146+33cba81e9",
|
30
|
+
"@atlaspack/rust": "3.0.1-dev.146+33cba81e9",
|
31
|
+
"@atlaspack/utils": "2.14.1-dev.146+33cba81e9",
|
32
|
+
"ncp": "^2.0.0"
|
32
33
|
},
|
33
34
|
"devDependencies": {
|
34
35
|
"idb": "^5.0.8"
|
35
36
|
},
|
36
37
|
"browser": {
|
37
|
-
"./src/IDBCache.js": "./src/IDBCache.browser.js"
|
38
|
-
"./src/LMDBCache.js": false
|
38
|
+
"./src/IDBCache.js": "./src/IDBCache.browser.js"
|
39
39
|
},
|
40
|
-
"gitHead": "
|
40
|
+
"gitHead": "33cba81e98b6eb098014f5a19d7932821b95f075"
|
41
41
|
}
|
package/src/FSCache.js
CHANGED
@@ -9,6 +9,8 @@ import stream from 'stream';
|
|
9
9
|
import path from 'path';
|
10
10
|
import {promisify} from 'util';
|
11
11
|
|
12
|
+
import {hashString} from '@atlaspack/rust';
|
13
|
+
import {getFeatureFlag} from '@atlaspack/feature-flags';
|
12
14
|
import logger from '@atlaspack/logger';
|
13
15
|
import {
|
14
16
|
deserialize,
|
@@ -51,6 +53,10 @@ export class FSCache implements Cache {
|
|
51
53
|
}
|
52
54
|
|
53
55
|
_getCachePath(cacheId: string): FilePath {
|
56
|
+
if (getFeatureFlag('cachePerformanceImprovements')) {
|
57
|
+
const cleanId = hashString(cacheId);
|
58
|
+
return path.join(this.dir, cleanId.slice(0, 2), cleanId.slice(2));
|
59
|
+
}
|
54
60
|
return path.join(this.dir, cacheId.slice(0, 2), cacheId.slice(2));
|
55
61
|
}
|
56
62
|
|
@@ -90,6 +96,9 @@ export class FSCache implements Cache {
|
|
90
96
|
}
|
91
97
|
|
92
98
|
#getFilePath(key: string, index: number): string {
|
99
|
+
if (getFeatureFlag('cachePerformanceImprovements')) {
|
100
|
+
return path.join(this.dir, `${hashString(key)}-${index}`);
|
101
|
+
}
|
93
102
|
return path.join(this.dir, `${key}-${index}`);
|
94
103
|
}
|
95
104
|
|
package/src/LMDBLiteCache.js
CHANGED
@@ -5,21 +5,23 @@ import {
|
|
5
5
|
registerSerializableClass,
|
6
6
|
serialize,
|
7
7
|
} from '@atlaspack/build-cache';
|
8
|
+
import {getFeatureFlag} from '@atlaspack/feature-flags';
|
8
9
|
import {Lmdb} from '@atlaspack/rust';
|
9
10
|
import type {FilePath} from '@atlaspack/types';
|
10
11
|
import type {Cache} from './types';
|
11
12
|
import type {Readable, Writable} from 'stream';
|
12
|
-
|
13
|
+
import fs from 'fs';
|
14
|
+
import ncp from 'ncp';
|
15
|
+
import {promisify} from 'util';
|
13
16
|
import stream from 'stream';
|
14
17
|
import path from 'path';
|
15
|
-
import {promisify} from 'util';
|
16
|
-
|
17
18
|
import {NodeFS} from '@atlaspack/fs';
|
18
|
-
|
19
19
|
// $FlowFixMe
|
20
20
|
import packageJson from '../package.json';
|
21
|
-
|
22
21
|
import {FSCache} from './FSCache';
|
22
|
+
import {instrumentAsync} from '@atlaspack/logger';
|
23
|
+
|
24
|
+
const ncpAsync = promisify(ncp);
|
23
25
|
|
24
26
|
interface DBOpenOptions {
|
25
27
|
name: string;
|
@@ -34,11 +36,14 @@ export class LmdbWrapper {
|
|
34
36
|
|
35
37
|
constructor(lmdb: Lmdb) {
|
36
38
|
this.lmdb = lmdb;
|
39
|
+
}
|
37
40
|
|
38
|
-
|
39
|
-
this
|
40
|
-
|
41
|
-
|
41
|
+
has(key: string): boolean {
|
42
|
+
return this.lmdb.hasSync(key);
|
43
|
+
}
|
44
|
+
|
45
|
+
async delete(key: string): Promise<void> {
|
46
|
+
await this.lmdb.delete(key);
|
42
47
|
}
|
43
48
|
|
44
49
|
get(key: string): Buffer | null {
|
@@ -51,7 +56,21 @@ export class LmdbWrapper {
|
|
51
56
|
await this.lmdb.put(key, buffer);
|
52
57
|
}
|
53
58
|
|
54
|
-
|
59
|
+
*keys(): Iterable<string> {
|
60
|
+
const PAGE_SIZE = 10000000;
|
61
|
+
|
62
|
+
let currentKeys = this.lmdb.keysSync(0, PAGE_SIZE);
|
63
|
+
while (currentKeys.length > 0) {
|
64
|
+
for (const key of currentKeys) {
|
65
|
+
yield key;
|
66
|
+
}
|
67
|
+
currentKeys = this.lmdb.keysSync(currentKeys.length, PAGE_SIZE);
|
68
|
+
}
|
69
|
+
}
|
70
|
+
|
71
|
+
compact(targetPath: string) {
|
72
|
+
this.lmdb.compact(targetPath);
|
73
|
+
}
|
55
74
|
}
|
56
75
|
|
57
76
|
export function open(
|
@@ -84,10 +103,15 @@ export class LMDBLiteCache implements Cache {
|
|
84
103
|
dir: FilePath;
|
85
104
|
store: LmdbWrapper;
|
86
105
|
fsCache: FSCache;
|
106
|
+
/**
|
107
|
+
* Directory where we store raw files.
|
108
|
+
*/
|
109
|
+
cacheFilesDirectory: FilePath;
|
87
110
|
|
88
111
|
constructor(cacheDir: FilePath) {
|
89
112
|
this.fs = new NodeFS();
|
90
113
|
this.dir = cacheDir;
|
114
|
+
this.cacheFilesDirectory = path.join(cacheDir, 'files');
|
91
115
|
this.fsCache = new FSCache(this.fs, cacheDir);
|
92
116
|
|
93
117
|
this.store = open(cacheDir, {
|
@@ -105,7 +129,10 @@ export class LMDBLiteCache implements Cache {
|
|
105
129
|
}
|
106
130
|
|
107
131
|
async ensure(): Promise<void> {
|
108
|
-
|
132
|
+
if (!getFeatureFlag('cachePerformanceImprovements')) {
|
133
|
+
await this.fsCache.ensure();
|
134
|
+
}
|
135
|
+
await this.fs.mkdirp(this.cacheFilesDirectory);
|
109
136
|
return Promise.resolve();
|
110
137
|
}
|
111
138
|
|
@@ -120,7 +147,7 @@ export class LMDBLiteCache implements Cache {
|
|
120
147
|
}
|
121
148
|
|
122
149
|
has(key: string): Promise<boolean> {
|
123
|
-
return Promise.resolve(this.store.
|
150
|
+
return Promise.resolve(this.store.has(key));
|
124
151
|
}
|
125
152
|
|
126
153
|
get<T>(key: string): Promise<?T> {
|
@@ -137,14 +164,24 @@ export class LMDBLiteCache implements Cache {
|
|
137
164
|
}
|
138
165
|
|
139
166
|
getStream(key: string): Readable {
|
140
|
-
|
167
|
+
if (!getFeatureFlag('cachePerformanceImprovements')) {
|
168
|
+
return this.fs.createReadStream(path.join(this.dir, key));
|
169
|
+
}
|
170
|
+
|
171
|
+
return fs.createReadStream(this.getFileKey(key));
|
141
172
|
}
|
142
173
|
|
143
|
-
setStream(key: string, stream: Readable): Promise<void> {
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
174
|
+
async setStream(key: string, stream: Readable): Promise<void> {
|
175
|
+
if (!getFeatureFlag('cachePerformanceImprovements')) {
|
176
|
+
return pipeline(
|
177
|
+
stream,
|
178
|
+
this.fs.createWriteStream(path.join(this.dir, key)),
|
179
|
+
);
|
180
|
+
}
|
181
|
+
|
182
|
+
const filePath = this.getFileKey(key);
|
183
|
+
await fs.promises.mkdir(path.dirname(filePath), {recursive: true});
|
184
|
+
return pipeline(stream, fs.createWriteStream(filePath));
|
148
185
|
}
|
149
186
|
|
150
187
|
// eslint-disable-next-line require-await
|
@@ -168,38 +205,107 @@ export class LMDBLiteCache implements Cache {
|
|
168
205
|
return Promise.resolve(this.store.get(key));
|
169
206
|
}
|
170
207
|
|
171
|
-
#getFilePath(key: string, index: number): string {
|
172
|
-
return path.join(this.dir, `${key}-${index}`);
|
173
|
-
}
|
174
|
-
|
175
208
|
hasLargeBlob(key: string): Promise<boolean> {
|
176
|
-
|
209
|
+
if (!getFeatureFlag('cachePerformanceImprovements')) {
|
210
|
+
return this.fsCache.hasLargeBlob(key);
|
211
|
+
}
|
212
|
+
|
213
|
+
return fs.promises
|
214
|
+
.access(this.getFileKey(key), fs.constants.F_OK)
|
215
|
+
.then(() => true)
|
216
|
+
.catch(() => false);
|
177
217
|
}
|
178
218
|
|
179
|
-
|
180
|
-
|
181
|
-
|
219
|
+
getLargeBlob(key: string): Promise<Buffer> {
|
220
|
+
if (!getFeatureFlag('cachePerformanceImprovements')) {
|
221
|
+
return this.fsCache.getLargeBlob(key);
|
222
|
+
}
|
223
|
+
return fs.promises.readFile(this.getFileKey(key));
|
182
224
|
}
|
183
225
|
|
184
|
-
// eslint-disable-next-line require-await
|
185
226
|
async setLargeBlob(
|
186
227
|
key: string,
|
187
228
|
contents: Buffer | string,
|
188
229
|
options?: {|signal?: AbortSignal|},
|
189
230
|
): Promise<void> {
|
190
|
-
|
231
|
+
if (!getFeatureFlag('cachePerformanceImprovements')) {
|
232
|
+
return this.fsCache.setLargeBlob(key, contents, options);
|
233
|
+
}
|
234
|
+
|
235
|
+
const targetPath = this.getFileKey(key);
|
236
|
+
await fs.promises.mkdir(path.dirname(targetPath), {recursive: true});
|
237
|
+
return fs.promises.writeFile(targetPath, contents);
|
191
238
|
}
|
192
239
|
|
240
|
+
/**
|
241
|
+
* @deprecated Use store.delete instead.
|
242
|
+
*/
|
193
243
|
deleteLargeBlob(key: string): Promise<void> {
|
194
|
-
|
244
|
+
if (!getFeatureFlag('cachePerformanceImprovements')) {
|
245
|
+
return this.fsCache.deleteLargeBlob(key);
|
246
|
+
}
|
247
|
+
|
248
|
+
return this.store.delete(key);
|
249
|
+
}
|
250
|
+
|
251
|
+
keys(): Iterable<string> {
|
252
|
+
return this.store.keys();
|
253
|
+
}
|
254
|
+
|
255
|
+
async compact(targetPath: string): Promise<void> {
|
256
|
+
await fs.promises.mkdir(targetPath, {recursive: true});
|
257
|
+
|
258
|
+
const files = await fs.promises.readdir(this.dir);
|
259
|
+
// copy all files except data.mdb and lock.mdb to the target path (recursive)
|
260
|
+
for (const file of files) {
|
261
|
+
const filePath = path.join(this.dir, file);
|
262
|
+
|
263
|
+
if (file === 'data.mdb' || file === 'lock.mdb') {
|
264
|
+
continue;
|
265
|
+
}
|
266
|
+
|
267
|
+
await ncpAsync(filePath, path.join(targetPath, file));
|
268
|
+
}
|
269
|
+
|
270
|
+
this.store.compact(path.join(targetPath, 'data.mdb'));
|
195
271
|
}
|
196
272
|
|
197
|
-
refresh(): void {
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
273
|
+
refresh(): void {}
|
274
|
+
|
275
|
+
/**
|
276
|
+
* Streams, packages are stored in files instead of LMDB.
|
277
|
+
*
|
278
|
+
* On this case, if a cache key happens to have a parent traversal, ../..
|
279
|
+
* it is treated specially
|
280
|
+
*
|
281
|
+
* That is, something/../something and something are meant to be different
|
282
|
+
* keys.
|
283
|
+
*
|
284
|
+
* Plus we do not want to store values outside of the cache directory.
|
285
|
+
*/
|
286
|
+
getFileKey(key: string): string {
|
287
|
+
const cleanKey = key
|
288
|
+
.split('/')
|
289
|
+
.map((part) => {
|
290
|
+
if (part === '..') {
|
291
|
+
return '$$__parent_dir$$';
|
292
|
+
}
|
293
|
+
return part;
|
294
|
+
})
|
295
|
+
.join('/');
|
296
|
+
return path.join(this.cacheFilesDirectory, cleanKey);
|
297
|
+
}
|
298
|
+
|
299
|
+
async clear(): Promise<void> {
|
300
|
+
await instrumentAsync('LMDBLiteCache::clear', async () => {
|
301
|
+
const keys = await this.keys();
|
302
|
+
for (const key of keys) {
|
303
|
+
await this.store.delete(key);
|
304
|
+
}
|
305
|
+
|
306
|
+
await this.fs.rimraf(this.cacheFilesDirectory);
|
307
|
+
await this.fs.mkdirp(this.cacheFilesDirectory);
|
308
|
+
});
|
203
309
|
}
|
204
310
|
}
|
205
311
|
|
package/src/index.js
CHANGED
@@ -1,20 +1,30 @@
|
|
1
1
|
// @flow
|
2
|
+
|
3
|
+
import * as fs from 'fs';
|
2
4
|
import * as path from 'path';
|
3
5
|
import {tmpdir} from 'os';
|
4
6
|
import {LMDBLiteCache} from '../src/index';
|
5
7
|
import {deserialize, serialize} from 'v8';
|
6
8
|
import assert from 'assert';
|
9
|
+
import {Worker} from 'worker_threads';
|
10
|
+
import {initializeMonitoring} from '@atlaspack/rust';
|
7
11
|
|
8
12
|
const cacheDir = path.join(tmpdir(), 'lmdb-lite-cache-tests');
|
9
13
|
|
10
14
|
describe('LMDBLiteCache', () => {
|
15
|
+
let cache;
|
16
|
+
|
17
|
+
beforeEach(async () => {
|
18
|
+
await fs.promises.rm(cacheDir, {recursive: true, force: true});
|
19
|
+
});
|
20
|
+
|
11
21
|
it('can be constructed', async () => {
|
12
|
-
|
22
|
+
cache = new LMDBLiteCache(cacheDir);
|
13
23
|
await cache.ensure();
|
14
24
|
});
|
15
25
|
|
16
26
|
it('can retrieve keys', async () => {
|
17
|
-
|
27
|
+
cache = new LMDBLiteCache(cacheDir);
|
18
28
|
await cache.ensure();
|
19
29
|
await cache.setBlob('key', Buffer.from(serialize({value: 42})));
|
20
30
|
const buffer = await cache.getBlob('key');
|
@@ -23,11 +33,211 @@ describe('LMDBLiteCache', () => {
|
|
23
33
|
});
|
24
34
|
|
25
35
|
it('can retrieve keys synchronously', async () => {
|
26
|
-
|
36
|
+
cache = new LMDBLiteCache(path.join(cacheDir, 'retrieve_keys_test'));
|
27
37
|
await cache.ensure();
|
28
|
-
cache.setBlob('key', Buffer.from(serialize({value: 42})));
|
38
|
+
await cache.setBlob('key', Buffer.from(serialize({value: 42})));
|
29
39
|
const buffer = cache.getBlobSync('key');
|
30
40
|
const result = deserialize(buffer);
|
31
41
|
assert.equal(result.value, 42);
|
32
42
|
});
|
43
|
+
|
44
|
+
it('can iterate over keys', async () => {
|
45
|
+
cache = new LMDBLiteCache(path.join(cacheDir, 'keys_test'));
|
46
|
+
await cache.ensure();
|
47
|
+
await cache.setBlob('key1', Buffer.from(serialize({value: 42})));
|
48
|
+
await cache.setBlob('key2', Buffer.from(serialize({value: 43})));
|
49
|
+
const keys = cache.keys();
|
50
|
+
assert.deepEqual(Array.from(keys), ['key1', 'key2']);
|
51
|
+
});
|
52
|
+
|
53
|
+
it('can compact databases', async () => {
|
54
|
+
cache = new LMDBLiteCache(path.join(cacheDir, 'compact_test'));
|
55
|
+
await cache.ensure();
|
56
|
+
await cache.setBlob('key1', Buffer.from(serialize({value: 42})));
|
57
|
+
await cache.setBlob('key2', Buffer.from(serialize({value: 43})));
|
58
|
+
await cache.compact(path.join(cacheDir, 'compact_test_compacted'));
|
59
|
+
|
60
|
+
cache = new LMDBLiteCache(path.join(cacheDir, 'compact_test_compacted'));
|
61
|
+
await cache.ensure();
|
62
|
+
const keys = cache.keys();
|
63
|
+
assert.deepEqual(Array.from(keys), ['key1', 'key2']);
|
64
|
+
});
|
65
|
+
|
66
|
+
describe('getFileKey', () => {
|
67
|
+
it('should return the correct key', () => {
|
68
|
+
const target = path.join(cacheDir, 'test-file-keys');
|
69
|
+
const cache = new LMDBLiteCache(target);
|
70
|
+
const key = cache.getFileKey('key');
|
71
|
+
assert.equal(key, path.join(target, 'files', 'key'));
|
72
|
+
});
|
73
|
+
|
74
|
+
it('should return the correct key for a key with a parent traversal', () => {
|
75
|
+
const target = path.join(cacheDir, 'test-parent-keys');
|
76
|
+
cache = new LMDBLiteCache(target);
|
77
|
+
const key = cache.getFileKey('../../key');
|
78
|
+
assert.equal(
|
79
|
+
key,
|
80
|
+
path.join(target, 'files', '$$__parent_dir$$/$$__parent_dir$$/key'),
|
81
|
+
);
|
82
|
+
});
|
83
|
+
});
|
84
|
+
|
85
|
+
it('can be closed and re-opened', async () => {
|
86
|
+
cache = new LMDBLiteCache(path.join(cacheDir, 'close_and_reopen_test'));
|
87
|
+
await cache.ensure();
|
88
|
+
await cache.setBlob('key', Buffer.from(serialize({value: 42})));
|
89
|
+
cache = new LMDBLiteCache(path.join(cacheDir, 'close_and_reopen_test'));
|
90
|
+
await cache.ensure();
|
91
|
+
const buffer = await cache.getBlob('key');
|
92
|
+
const result = deserialize(buffer);
|
93
|
+
assert.equal(result.value, 42);
|
94
|
+
});
|
95
|
+
|
96
|
+
it('should NOT fail when trying to open the same database twice', async () => {
|
97
|
+
const testDir = path.join(cacheDir, 'double_open_test');
|
98
|
+
const cache1 = new LMDBLiteCache(testDir);
|
99
|
+
await cache1.ensure();
|
100
|
+
|
101
|
+
assert.doesNotThrow(() => {
|
102
|
+
new LMDBLiteCache(testDir);
|
103
|
+
});
|
104
|
+
});
|
105
|
+
|
106
|
+
it('should NOT fail when trying to open after GC', async () => {
|
107
|
+
const testDir = path.join(cacheDir, 'gc_test');
|
108
|
+
|
109
|
+
let cache1 = new LMDBLiteCache(testDir);
|
110
|
+
await cache1.ensure();
|
111
|
+
await cache1.setBlob('key', Buffer.from(serialize({value: 42})));
|
112
|
+
|
113
|
+
cache1 = null;
|
114
|
+
|
115
|
+
if (global.gc) {
|
116
|
+
global.gc();
|
117
|
+
}
|
118
|
+
|
119
|
+
assert.doesNotThrow(() => {
|
120
|
+
new LMDBLiteCache(testDir);
|
121
|
+
});
|
122
|
+
});
|
123
|
+
|
124
|
+
it('should handle rapid open/close cycles', async () => {
|
125
|
+
const testDir = path.join(cacheDir, 'rapid_cycles_test');
|
126
|
+
|
127
|
+
for (let i = 0; i < 10; i++) {
|
128
|
+
const cache = new LMDBLiteCache(testDir);
|
129
|
+
await cache.ensure();
|
130
|
+
await cache.setBlob(`key${i}`, Buffer.from(serialize({value: i})));
|
131
|
+
|
132
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
133
|
+
}
|
134
|
+
|
135
|
+
const finalCache = new LMDBLiteCache(testDir);
|
136
|
+
await finalCache.ensure();
|
137
|
+
const buffer = await finalCache.getBlob('key9');
|
138
|
+
const result = deserialize(buffer);
|
139
|
+
assert.equal(result.value, 9);
|
140
|
+
});
|
141
|
+
|
142
|
+
it('should work when there are multiple node.js worker threads accessing the same database', async function () {
|
143
|
+
this.timeout(40000);
|
144
|
+
|
145
|
+
try {
|
146
|
+
initializeMonitoring();
|
147
|
+
} catch (error) {
|
148
|
+
/* empty */
|
149
|
+
}
|
150
|
+
|
151
|
+
const testDir = path.join(cacheDir, 'worker_threads_test');
|
152
|
+
|
153
|
+
let cache = new LMDBLiteCache(testDir);
|
154
|
+
await cache.set('main_thread_key', {
|
155
|
+
mainThreadId: 0,
|
156
|
+
hello: 'world',
|
157
|
+
});
|
158
|
+
setTimeout(() => {
|
159
|
+
cache = null;
|
160
|
+
|
161
|
+
if (global.gc) {
|
162
|
+
global.gc();
|
163
|
+
}
|
164
|
+
}, Math.random() * 300);
|
165
|
+
|
166
|
+
const numWorkers = 10;
|
167
|
+
|
168
|
+
const workers = [];
|
169
|
+
const responsePromises = [];
|
170
|
+
for (let i = 0; i < numWorkers; i++) {
|
171
|
+
const worker = new Worker(path.join(__dirname, 'workerThreadsTest.js'), {
|
172
|
+
workerData: {
|
173
|
+
cacheDir: testDir,
|
174
|
+
},
|
175
|
+
});
|
176
|
+
workers.push(worker);
|
177
|
+
|
178
|
+
const responsePromise = new Promise((resolve, reject) => {
|
179
|
+
worker.addListener('error', (error: Error) => {
|
180
|
+
reject(error);
|
181
|
+
});
|
182
|
+
worker.addListener('message', (message) => {
|
183
|
+
resolve(message);
|
184
|
+
});
|
185
|
+
});
|
186
|
+
|
187
|
+
worker.addListener('message', (message) => {
|
188
|
+
// eslint-disable-next-line no-console
|
189
|
+
console.log('Worker message', message);
|
190
|
+
});
|
191
|
+
worker.addListener('online', () => {
|
192
|
+
worker.postMessage({
|
193
|
+
type: 'go',
|
194
|
+
});
|
195
|
+
});
|
196
|
+
|
197
|
+
responsePromises.push(responsePromise);
|
198
|
+
}
|
199
|
+
|
200
|
+
// eslint-disable-next-line no-console
|
201
|
+
console.log('Waiting for responses');
|
202
|
+
const responses = await Promise.all(responsePromises);
|
203
|
+
|
204
|
+
// eslint-disable-next-line no-console
|
205
|
+
console.log('Responses received');
|
206
|
+
for (const [index, response] of responses.entries()) {
|
207
|
+
const worker = workers[index];
|
208
|
+
|
209
|
+
assert.deepEqual(
|
210
|
+
response,
|
211
|
+
{
|
212
|
+
mainThreadData: {
|
213
|
+
mainThreadId: 0,
|
214
|
+
hello: 'world',
|
215
|
+
},
|
216
|
+
workerId: worker.threadId,
|
217
|
+
},
|
218
|
+
`worker_${index} - Worker ${worker.threadId} should have received the correct data`,
|
219
|
+
);
|
220
|
+
}
|
221
|
+
|
222
|
+
// eslint-disable-next-line no-console
|
223
|
+
console.log('Getting main thread key');
|
224
|
+
cache = new LMDBLiteCache(testDir);
|
225
|
+
const data = await cache?.get('main_thread_key');
|
226
|
+
assert.deepEqual(data, {
|
227
|
+
mainThreadId: 0,
|
228
|
+
hello: 'world',
|
229
|
+
});
|
230
|
+
|
231
|
+
// eslint-disable-next-line no-console
|
232
|
+
console.log('Getting worker keys');
|
233
|
+
for (const worker of workers) {
|
234
|
+
const data = await cache?.get(`worker_key/${worker.threadId}`);
|
235
|
+
assert.deepEqual(data, {
|
236
|
+
workerId: worker.threadId,
|
237
|
+
});
|
238
|
+
|
239
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
240
|
+
worker.terminate();
|
241
|
+
}
|
242
|
+
});
|
33
243
|
});
|
@@ -0,0 +1,42 @@
|
|
1
|
+
/* eslint-disable no-inner-declarations */
|
2
|
+
|
3
|
+
require('@atlaspack/babel-register');
|
4
|
+
const {
|
5
|
+
workerData,
|
6
|
+
threadId,
|
7
|
+
parentPort,
|
8
|
+
isMainThread,
|
9
|
+
} = require('worker_threads');
|
10
|
+
const {LMDBLiteCache} = require('../src/index');
|
11
|
+
|
12
|
+
if (!isMainThread) {
|
13
|
+
const cache = new LMDBLiteCache(workerData.cacheDir);
|
14
|
+
|
15
|
+
async function onMessage() {
|
16
|
+
try {
|
17
|
+
cache.set(`worker_key/${threadId}`, {
|
18
|
+
workerId: threadId,
|
19
|
+
});
|
20
|
+
|
21
|
+
const data = await cache.get('main_thread_key');
|
22
|
+
|
23
|
+
parentPort.postMessage({
|
24
|
+
mainThreadData: data,
|
25
|
+
workerId: threadId,
|
26
|
+
});
|
27
|
+
|
28
|
+
setTimeout(() => {
|
29
|
+
parentPort.postMessage({
|
30
|
+
type: 'close',
|
31
|
+
workerId: threadId,
|
32
|
+
});
|
33
|
+
}, Math.random() * 200);
|
34
|
+
} catch (error) {
|
35
|
+
parentPort.postMessage({
|
36
|
+
error: error.message,
|
37
|
+
});
|
38
|
+
}
|
39
|
+
}
|
40
|
+
|
41
|
+
parentPort.on('message', onMessage);
|
42
|
+
}
|
package/lib/LMDBCache.js
DELETED
@@ -1,136 +0,0 @@
|
|
1
|
-
"use strict";
|
2
|
-
|
3
|
-
Object.defineProperty(exports, "__esModule", {
|
4
|
-
value: true
|
5
|
-
});
|
6
|
-
exports.LMDBCache = void 0;
|
7
|
-
function _stream() {
|
8
|
-
const data = _interopRequireDefault(require("stream"));
|
9
|
-
_stream = function () {
|
10
|
-
return data;
|
11
|
-
};
|
12
|
-
return data;
|
13
|
-
}
|
14
|
-
function _path() {
|
15
|
-
const data = _interopRequireDefault(require("path"));
|
16
|
-
_path = function () {
|
17
|
-
return data;
|
18
|
-
};
|
19
|
-
return data;
|
20
|
-
}
|
21
|
-
function _util() {
|
22
|
-
const data = require("util");
|
23
|
-
_util = function () {
|
24
|
-
return data;
|
25
|
-
};
|
26
|
-
return data;
|
27
|
-
}
|
28
|
-
function _buildCache() {
|
29
|
-
const data = require("@atlaspack/build-cache");
|
30
|
-
_buildCache = function () {
|
31
|
-
return data;
|
32
|
-
};
|
33
|
-
return data;
|
34
|
-
}
|
35
|
-
function _fs() {
|
36
|
-
const data = require("@atlaspack/fs");
|
37
|
-
_fs = function () {
|
38
|
-
return data;
|
39
|
-
};
|
40
|
-
return data;
|
41
|
-
}
|
42
|
-
function _lmdb() {
|
43
|
-
const data = _interopRequireDefault(require("lmdb"));
|
44
|
-
_lmdb = function () {
|
45
|
-
return data;
|
46
|
-
};
|
47
|
-
return data;
|
48
|
-
}
|
49
|
-
var _package = _interopRequireDefault(require("../package.json"));
|
50
|
-
var _FSCache = require("./FSCache");
|
51
|
-
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
52
|
-
// flowlint-next-line untyped-import:off
|
53
|
-
// $FlowFixMe
|
54
|
-
const pipeline = (0, _util().promisify)(_stream().default.pipeline);
|
55
|
-
class LMDBCache {
|
56
|
-
// $FlowFixMe
|
57
|
-
|
58
|
-
constructor(cacheDir) {
|
59
|
-
this.fs = new (_fs().NodeFS)();
|
60
|
-
this.dir = cacheDir;
|
61
|
-
this.fsCache = new _FSCache.FSCache(this.fs, cacheDir);
|
62
|
-
this.store = _lmdb().default.open(cacheDir, {
|
63
|
-
name: 'parcel-cache',
|
64
|
-
encoding: 'binary',
|
65
|
-
compression: true
|
66
|
-
});
|
67
|
-
}
|
68
|
-
ensure() {
|
69
|
-
return Promise.resolve();
|
70
|
-
}
|
71
|
-
serialize() {
|
72
|
-
return {
|
73
|
-
dir: this.dir
|
74
|
-
};
|
75
|
-
}
|
76
|
-
static deserialize(opts) {
|
77
|
-
return new LMDBCache(opts.dir);
|
78
|
-
}
|
79
|
-
has(key) {
|
80
|
-
return Promise.resolve(this.store.get(key) != null);
|
81
|
-
}
|
82
|
-
get(key) {
|
83
|
-
let data = this.store.get(key);
|
84
|
-
if (data == null) {
|
85
|
-
return Promise.resolve(null);
|
86
|
-
}
|
87
|
-
return Promise.resolve((0, _buildCache().deserialize)(data));
|
88
|
-
}
|
89
|
-
async set(key, value) {
|
90
|
-
await this.setBlob(key, (0, _buildCache().serialize)(value));
|
91
|
-
}
|
92
|
-
getStream(key) {
|
93
|
-
return this.fs.createReadStream(_path().default.join(this.dir, key));
|
94
|
-
}
|
95
|
-
setStream(key, stream) {
|
96
|
-
return pipeline(stream, this.fs.createWriteStream(_path().default.join(this.dir, key)));
|
97
|
-
}
|
98
|
-
getBlob(key) {
|
99
|
-
let buffer = this.store.get(key);
|
100
|
-
return buffer != null ? Promise.resolve(buffer) : Promise.reject(new Error(`Key ${key} not found in cache`));
|
101
|
-
}
|
102
|
-
async setBlob(key, contents) {
|
103
|
-
await this.store.put(key, contents);
|
104
|
-
}
|
105
|
-
getBuffer(key) {
|
106
|
-
return Promise.resolve(this.store.get(key));
|
107
|
-
}
|
108
|
-
#getFilePath(key, index) {
|
109
|
-
return _path().default.join(this.dir, `${key}-${index}`);
|
110
|
-
}
|
111
|
-
hasLargeBlob(key) {
|
112
|
-
return this.fs.exists(this.#getFilePath(key, 0));
|
113
|
-
}
|
114
|
-
|
115
|
-
// eslint-disable-next-line require-await
|
116
|
-
async getLargeBlob(key) {
|
117
|
-
return this.fsCache.getLargeBlob(key);
|
118
|
-
}
|
119
|
-
|
120
|
-
// eslint-disable-next-line require-await
|
121
|
-
async setLargeBlob(key, contents, options) {
|
122
|
-
return this.fsCache.setLargeBlob(key, contents, options);
|
123
|
-
}
|
124
|
-
deleteLargeBlob(key) {
|
125
|
-
return this.fsCache.deleteLargeBlob(key);
|
126
|
-
}
|
127
|
-
refresh() {
|
128
|
-
// Reset the read transaction for the store. This guarantees that
|
129
|
-
// the next read will see the latest changes to the store.
|
130
|
-
// Useful in scenarios where reads and writes are multi-threaded.
|
131
|
-
// See https://github.com/kriszyp/lmdb-js#resetreadtxn-void
|
132
|
-
this.store.resetReadTxn();
|
133
|
-
}
|
134
|
-
}
|
135
|
-
exports.LMDBCache = LMDBCache;
|
136
|
-
(0, _buildCache().registerSerializableClass)(`${_package.default.version}:LMDBCache`, LMDBCache);
|
package/src/LMDBCache.js
DELETED
@@ -1,139 +0,0 @@
|
|
1
|
-
// @flow strict-local
|
2
|
-
import type {FilePath} from '@atlaspack/types';
|
3
|
-
import type {Cache} from './types';
|
4
|
-
import type {Readable, Writable} from 'stream';
|
5
|
-
|
6
|
-
import stream from 'stream';
|
7
|
-
import path from 'path';
|
8
|
-
import {promisify} from 'util';
|
9
|
-
|
10
|
-
import {
|
11
|
-
deserialize,
|
12
|
-
registerSerializableClass,
|
13
|
-
serialize,
|
14
|
-
} from '@atlaspack/build-cache';
|
15
|
-
import {NodeFS} from '@atlaspack/fs';
|
16
|
-
// flowlint-next-line untyped-import:off
|
17
|
-
import lmdb from 'lmdb';
|
18
|
-
|
19
|
-
// $FlowFixMe
|
20
|
-
import packageJson from '../package.json';
|
21
|
-
|
22
|
-
import {FSCache} from './FSCache';
|
23
|
-
|
24
|
-
const pipeline: (Readable, Writable) => Promise<void> = promisify(
|
25
|
-
stream.pipeline,
|
26
|
-
);
|
27
|
-
|
28
|
-
export class LMDBCache implements Cache {
|
29
|
-
fs: NodeFS;
|
30
|
-
dir: FilePath;
|
31
|
-
// $FlowFixMe
|
32
|
-
store: any;
|
33
|
-
fsCache: FSCache;
|
34
|
-
|
35
|
-
constructor(cacheDir: FilePath) {
|
36
|
-
this.fs = new NodeFS();
|
37
|
-
this.dir = cacheDir;
|
38
|
-
this.fsCache = new FSCache(this.fs, cacheDir);
|
39
|
-
|
40
|
-
this.store = lmdb.open(cacheDir, {
|
41
|
-
name: 'parcel-cache',
|
42
|
-
encoding: 'binary',
|
43
|
-
compression: true,
|
44
|
-
});
|
45
|
-
}
|
46
|
-
|
47
|
-
ensure(): Promise<void> {
|
48
|
-
return Promise.resolve();
|
49
|
-
}
|
50
|
-
|
51
|
-
serialize(): {|dir: FilePath|} {
|
52
|
-
return {
|
53
|
-
dir: this.dir,
|
54
|
-
};
|
55
|
-
}
|
56
|
-
|
57
|
-
static deserialize(opts: {|dir: FilePath|}): LMDBCache {
|
58
|
-
return new LMDBCache(opts.dir);
|
59
|
-
}
|
60
|
-
|
61
|
-
has(key: string): Promise<boolean> {
|
62
|
-
return Promise.resolve(this.store.get(key) != null);
|
63
|
-
}
|
64
|
-
|
65
|
-
get<T>(key: string): Promise<?T> {
|
66
|
-
let data = this.store.get(key);
|
67
|
-
if (data == null) {
|
68
|
-
return Promise.resolve(null);
|
69
|
-
}
|
70
|
-
|
71
|
-
return Promise.resolve(deserialize(data));
|
72
|
-
}
|
73
|
-
|
74
|
-
async set(key: string, value: mixed): Promise<void> {
|
75
|
-
await this.setBlob(key, serialize(value));
|
76
|
-
}
|
77
|
-
|
78
|
-
getStream(key: string): Readable {
|
79
|
-
return this.fs.createReadStream(path.join(this.dir, key));
|
80
|
-
}
|
81
|
-
|
82
|
-
setStream(key: string, stream: Readable): Promise<void> {
|
83
|
-
return pipeline(
|
84
|
-
stream,
|
85
|
-
this.fs.createWriteStream(path.join(this.dir, key)),
|
86
|
-
);
|
87
|
-
}
|
88
|
-
|
89
|
-
getBlob(key: string): Promise<Buffer> {
|
90
|
-
let buffer = this.store.get(key);
|
91
|
-
return buffer != null
|
92
|
-
? Promise.resolve(buffer)
|
93
|
-
: Promise.reject(new Error(`Key ${key} not found in cache`));
|
94
|
-
}
|
95
|
-
|
96
|
-
async setBlob(key: string, contents: Buffer | string): Promise<void> {
|
97
|
-
await this.store.put(key, contents);
|
98
|
-
}
|
99
|
-
|
100
|
-
getBuffer(key: string): Promise<?Buffer> {
|
101
|
-
return Promise.resolve(this.store.get(key));
|
102
|
-
}
|
103
|
-
|
104
|
-
#getFilePath(key: string, index: number): string {
|
105
|
-
return path.join(this.dir, `${key}-${index}`);
|
106
|
-
}
|
107
|
-
|
108
|
-
hasLargeBlob(key: string): Promise<boolean> {
|
109
|
-
return this.fs.exists(this.#getFilePath(key, 0));
|
110
|
-
}
|
111
|
-
|
112
|
-
// eslint-disable-next-line require-await
|
113
|
-
async getLargeBlob(key: string): Promise<Buffer> {
|
114
|
-
return this.fsCache.getLargeBlob(key);
|
115
|
-
}
|
116
|
-
|
117
|
-
// eslint-disable-next-line require-await
|
118
|
-
async setLargeBlob(
|
119
|
-
key: string,
|
120
|
-
contents: Buffer | string,
|
121
|
-
options?: {|signal?: AbortSignal|},
|
122
|
-
): Promise<void> {
|
123
|
-
return this.fsCache.setLargeBlob(key, contents, options);
|
124
|
-
}
|
125
|
-
|
126
|
-
deleteLargeBlob(key: string): Promise<void> {
|
127
|
-
return this.fsCache.deleteLargeBlob(key);
|
128
|
-
}
|
129
|
-
|
130
|
-
refresh(): void {
|
131
|
-
// Reset the read transaction for the store. This guarantees that
|
132
|
-
// the next read will see the latest changes to the store.
|
133
|
-
// Useful in scenarios where reads and writes are multi-threaded.
|
134
|
-
// See https://github.com/kriszyp/lmdb-js#resetreadtxn-void
|
135
|
-
this.store.resetReadTxn();
|
136
|
-
}
|
137
|
-
}
|
138
|
-
|
139
|
-
registerSerializableClass(`${packageJson.version}:LMDBCache`, LMDBCache);
|