@atlaspack/cache 3.2.3 → 3.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/lib/FSCache.js +21 -0
- package/lib/LMDBLiteCache.js +65 -23
- package/package.json +6 -6
- package/src/FSCache.js +9 -0
- package/src/LMDBLiteCache.js +70 -25
- package/test/LMDBLiteCache.test.js +180 -6
- package/test/workerThreadsTest.js +42 -0
package/CHANGELOG.md
CHANGED
@@ -1,5 +1,28 @@
|
|
1
1
|
# @atlaspack/cache
|
2
2
|
|
3
|
+
## 3.2.5
|
4
|
+
|
5
|
+
### Patch Changes
|
6
|
+
|
7
|
+
- Updated dependencies [[`0999fb7`](https://github.com/atlassian-labs/atlaspack/commit/0999fb78da519a6c7582d212883e515fcf6c1252), [`35fdd4b`](https://github.com/atlassian-labs/atlaspack/commit/35fdd4b52da0af20f74667f7b8adfb2f90279b7c), [`6dd4ccb`](https://github.com/atlassian-labs/atlaspack/commit/6dd4ccb753541de32322d881f973d571dd57e4ca)]:
|
8
|
+
- @atlaspack/fs@2.15.5
|
9
|
+
- @atlaspack/rust@3.3.5
|
10
|
+
- @atlaspack/logger@2.14.10
|
11
|
+
- @atlaspack/utils@2.14.10
|
12
|
+
|
13
|
+
## 3.2.4
|
14
|
+
|
15
|
+
### Patch Changes
|
16
|
+
|
17
|
+
- [#583](https://github.com/atlassian-labs/atlaspack/pull/583) [`124b7ff`](https://github.com/atlassian-labs/atlaspack/commit/124b7fff44f71aac9fbad289a9a9509b3dfc9aaa) Thanks [@yamadapc](https://github.com/yamadapc)! - Fix problem where cache writes could start to fail during a V3 build
|
18
|
+
|
19
|
+
- Updated dependencies [[`124b7ff`](https://github.com/atlassian-labs/atlaspack/commit/124b7fff44f71aac9fbad289a9a9509b3dfc9aaa), [`e052521`](https://github.com/atlassian-labs/atlaspack/commit/e0525210850ed1606146eb86991049cf567c5dec), [`15c6d70`](https://github.com/atlassian-labs/atlaspack/commit/15c6d7000bd89da876bc590aa75b17a619a41896), [`e4d966c`](https://github.com/atlassian-labs/atlaspack/commit/e4d966c3c9c4292c5013372ae65b10d19d4bacc6), [`209692f`](https://github.com/atlassian-labs/atlaspack/commit/209692ffb11eae103a0d65c5e1118a5aa1625818), [`42a775d`](https://github.com/atlassian-labs/atlaspack/commit/42a775de8eec638ad188f3271964170d8c04d84b), [`29c2f10`](https://github.com/atlassian-labs/atlaspack/commit/29c2f106de9679adfb5afa04e1910471dc65a427), [`f4da1e1`](https://github.com/atlassian-labs/atlaspack/commit/f4da1e120e73eeb5e8b8927f05e88f04d6148c7b), [`1ef91fc`](https://github.com/atlassian-labs/atlaspack/commit/1ef91fcc863fdd2831511937083dbbc1263b3d9d)]:
|
20
|
+
- @atlaspack/rust@3.3.4
|
21
|
+
- @atlaspack/fs@2.15.4
|
22
|
+
- @atlaspack/feature-flags@2.16.0
|
23
|
+
- @atlaspack/logger@2.14.9
|
24
|
+
- @atlaspack/utils@2.14.9
|
25
|
+
|
3
26
|
## 3.2.3
|
4
27
|
|
5
28
|
### Patch Changes
|
package/lib/FSCache.js
CHANGED
@@ -25,6 +25,20 @@ function _util() {
|
|
25
25
|
};
|
26
26
|
return data;
|
27
27
|
}
|
28
|
+
function _rust() {
|
29
|
+
const data = require("@atlaspack/rust");
|
30
|
+
_rust = function () {
|
31
|
+
return data;
|
32
|
+
};
|
33
|
+
return data;
|
34
|
+
}
|
35
|
+
function _featureFlags() {
|
36
|
+
const data = require("@atlaspack/feature-flags");
|
37
|
+
_featureFlags = function () {
|
38
|
+
return data;
|
39
|
+
};
|
40
|
+
return data;
|
41
|
+
}
|
28
42
|
function _logger() {
|
29
43
|
const data = _interopRequireDefault(require("@atlaspack/logger"));
|
30
44
|
_logger = function () {
|
@@ -62,6 +76,10 @@ class FSCache {
|
|
62
76
|
await Promise.all(dirPromises);
|
63
77
|
}
|
64
78
|
_getCachePath(cacheId) {
|
79
|
+
if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
80
|
+
const cleanId = (0, _rust().hashString)(cacheId);
|
81
|
+
return _path().default.join(this.dir, cleanId.slice(0, 2), cleanId.slice(2));
|
82
|
+
}
|
65
83
|
return _path().default.join(this.dir, cacheId.slice(0, 2), cacheId.slice(2));
|
66
84
|
}
|
67
85
|
getStream(key) {
|
@@ -91,6 +109,9 @@ class FSCache {
|
|
91
109
|
}
|
92
110
|
}
|
93
111
|
#getFilePath(key, index) {
|
112
|
+
if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
113
|
+
return _path().default.join(this.dir, `${(0, _rust().hashString)(key)}-${index}`);
|
114
|
+
}
|
94
115
|
return _path().default.join(this.dir, `${key}-${index}`);
|
95
116
|
}
|
96
117
|
async #unlinkChunks(key, index) {
|
package/lib/LMDBLiteCache.js
CHANGED
@@ -70,17 +70,19 @@ function _fs2() {
|
|
70
70
|
}
|
71
71
|
var _package = _interopRequireDefault(require("../package.json"));
|
72
72
|
var _FSCache = require("./FSCache");
|
73
|
+
function _logger() {
|
74
|
+
const data = require("@atlaspack/logger");
|
75
|
+
_logger = function () {
|
76
|
+
return data;
|
77
|
+
};
|
78
|
+
return data;
|
79
|
+
}
|
73
80
|
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
74
81
|
// $FlowFixMe
|
75
82
|
const ncpAsync = (0, _util().promisify)(_ncp().default);
|
76
83
|
class LmdbWrapper {
|
77
84
|
constructor(lmdb) {
|
78
85
|
this.lmdb = lmdb;
|
79
|
-
|
80
|
-
// $FlowFixMe
|
81
|
-
this[Symbol.dispose] = () => {
|
82
|
-
this.lmdb.close();
|
83
|
-
};
|
84
86
|
}
|
85
87
|
has(key) {
|
86
88
|
return this.lmdb.hasSync(key);
|
@@ -121,9 +123,14 @@ function open(directory
|
|
121
123
|
}
|
122
124
|
const pipeline = (0, _util().promisify)(_stream().default.pipeline);
|
123
125
|
class LMDBLiteCache {
|
126
|
+
/**
|
127
|
+
* Directory where we store raw files.
|
128
|
+
*/
|
129
|
+
|
124
130
|
constructor(cacheDir) {
|
125
131
|
this.fs = new (_fs2().NodeFS)();
|
126
132
|
this.dir = cacheDir;
|
133
|
+
this.cacheFilesDirectory = _path().default.join(cacheDir, 'files');
|
127
134
|
this.fsCache = new _FSCache.FSCache(this.fs, cacheDir);
|
128
135
|
this.store = open(cacheDir, {
|
129
136
|
name: 'parcel-cache',
|
@@ -142,6 +149,7 @@ class LMDBLiteCache {
|
|
142
149
|
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
143
150
|
await this.fsCache.ensure();
|
144
151
|
}
|
152
|
+
await this.fs.mkdirp(this.cacheFilesDirectory);
|
145
153
|
return Promise.resolve();
|
146
154
|
}
|
147
155
|
serialize() {
|
@@ -166,10 +174,20 @@ class LMDBLiteCache {
|
|
166
174
|
await this.setBlob(key, (0, _buildCache().serialize)(value));
|
167
175
|
}
|
168
176
|
getStream(key) {
|
169
|
-
|
177
|
+
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
178
|
+
return this.fs.createReadStream(_path().default.join(this.dir, key));
|
179
|
+
}
|
180
|
+
return _fs().default.createReadStream(this.getFileKey(key));
|
170
181
|
}
|
171
|
-
setStream(key, stream) {
|
172
|
-
|
182
|
+
async setStream(key, stream) {
|
183
|
+
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
184
|
+
return pipeline(stream, this.fs.createWriteStream(_path().default.join(this.dir, key)));
|
185
|
+
}
|
186
|
+
const filePath = this.getFileKey(key);
|
187
|
+
await _fs().default.promises.mkdir(_path().default.dirname(filePath), {
|
188
|
+
recursive: true
|
189
|
+
});
|
190
|
+
return pipeline(stream, _fs().default.createWriteStream(filePath));
|
173
191
|
}
|
174
192
|
|
175
193
|
// eslint-disable-next-line require-await
|
@@ -189,34 +207,27 @@ class LMDBLiteCache {
|
|
189
207
|
getBuffer(key) {
|
190
208
|
return Promise.resolve(this.store.get(key));
|
191
209
|
}
|
192
|
-
#getFilePath(key, index) {
|
193
|
-
return _path().default.join(this.dir, `${key}-${index}`);
|
194
|
-
}
|
195
210
|
hasLargeBlob(key) {
|
196
211
|
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
197
212
|
return this.fsCache.hasLargeBlob(key);
|
198
213
|
}
|
199
|
-
return this.
|
214
|
+
return _fs().default.promises.access(this.getFileKey(key), _fs().default.constants.F_OK).then(() => true).catch(() => false);
|
200
215
|
}
|
201
|
-
|
202
|
-
/**
|
203
|
-
* @deprecated Use getBlob instead.
|
204
|
-
*/
|
205
216
|
getLargeBlob(key) {
|
206
217
|
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
207
218
|
return this.fsCache.getLargeBlob(key);
|
208
219
|
}
|
209
|
-
return
|
220
|
+
return _fs().default.promises.readFile(this.getFileKey(key));
|
210
221
|
}
|
211
|
-
|
212
|
-
/**
|
213
|
-
* @deprecated Use setBlob instead.
|
214
|
-
*/
|
215
|
-
setLargeBlob(key, contents, options) {
|
222
|
+
async setLargeBlob(key, contents, options) {
|
216
223
|
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
217
224
|
return this.fsCache.setLargeBlob(key, contents, options);
|
218
225
|
}
|
219
|
-
|
226
|
+
const targetPath = this.getFileKey(key);
|
227
|
+
await _fs().default.promises.mkdir(_path().default.dirname(targetPath), {
|
228
|
+
recursive: true
|
229
|
+
});
|
230
|
+
return _fs().default.promises.writeFile(targetPath, contents);
|
220
231
|
}
|
221
232
|
|
222
233
|
/**
|
@@ -247,6 +258,37 @@ class LMDBLiteCache {
|
|
247
258
|
this.store.compact(_path().default.join(targetPath, 'data.mdb'));
|
248
259
|
}
|
249
260
|
refresh() {}
|
261
|
+
|
262
|
+
/**
|
263
|
+
* Streams, packages are stored in files instead of LMDB.
|
264
|
+
*
|
265
|
+
* On this case, if a cache key happens to have a parent traversal, ../..
|
266
|
+
* it is treated specially
|
267
|
+
*
|
268
|
+
* That is, something/../something and something are meant to be different
|
269
|
+
* keys.
|
270
|
+
*
|
271
|
+
* Plus we do not want to store values outside of the cache directory.
|
272
|
+
*/
|
273
|
+
getFileKey(key) {
|
274
|
+
const cleanKey = key.split('/').map(part => {
|
275
|
+
if (part === '..') {
|
276
|
+
return '$$__parent_dir$$';
|
277
|
+
}
|
278
|
+
return part;
|
279
|
+
}).join('/');
|
280
|
+
return _path().default.join(this.cacheFilesDirectory, cleanKey);
|
281
|
+
}
|
282
|
+
async clear() {
|
283
|
+
await (0, _logger().instrumentAsync)('LMDBLiteCache::clear', async () => {
|
284
|
+
const keys = await this.keys();
|
285
|
+
for (const key of keys) {
|
286
|
+
await this.store.delete(key);
|
287
|
+
}
|
288
|
+
await this.fs.rimraf(this.cacheFilesDirectory);
|
289
|
+
await this.fs.mkdirp(this.cacheFilesDirectory);
|
290
|
+
});
|
291
|
+
}
|
250
292
|
}
|
251
293
|
exports.LMDBLiteCache = LMDBLiteCache;
|
252
294
|
(0, _buildCache().registerSerializableClass)(`${_package.default.version}:LMDBLiteCache`, LMDBLiteCache);
|
package/package.json
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
{
|
2
2
|
"name": "@atlaspack/cache",
|
3
3
|
"description": "Interface for defining caches and file-system, IDB and LMDB implementations.",
|
4
|
-
"version": "3.2.
|
4
|
+
"version": "3.2.5",
|
5
5
|
"license": "(MIT OR Apache-2.0)",
|
6
6
|
"type": "commonjs",
|
7
7
|
"publishConfig": {
|
@@ -24,11 +24,11 @@
|
|
24
24
|
},
|
25
25
|
"dependencies": {
|
26
26
|
"@atlaspack/build-cache": "2.13.3",
|
27
|
-
"@atlaspack/feature-flags": "2.
|
28
|
-
"@atlaspack/fs": "2.15.
|
29
|
-
"@atlaspack/logger": "2.14.
|
30
|
-
"@atlaspack/rust": "3.3.
|
31
|
-
"@atlaspack/utils": "2.14.
|
27
|
+
"@atlaspack/feature-flags": "2.16.0",
|
28
|
+
"@atlaspack/fs": "2.15.5",
|
29
|
+
"@atlaspack/logger": "2.14.10",
|
30
|
+
"@atlaspack/rust": "3.3.5",
|
31
|
+
"@atlaspack/utils": "2.14.10",
|
32
32
|
"ncp": "^2.0.0"
|
33
33
|
},
|
34
34
|
"devDependencies": {
|
package/src/FSCache.js
CHANGED
@@ -9,6 +9,8 @@ import stream from 'stream';
|
|
9
9
|
import path from 'path';
|
10
10
|
import {promisify} from 'util';
|
11
11
|
|
12
|
+
import {hashString} from '@atlaspack/rust';
|
13
|
+
import {getFeatureFlag} from '@atlaspack/feature-flags';
|
12
14
|
import logger from '@atlaspack/logger';
|
13
15
|
import {
|
14
16
|
deserialize,
|
@@ -51,6 +53,10 @@ export class FSCache implements Cache {
|
|
51
53
|
}
|
52
54
|
|
53
55
|
_getCachePath(cacheId: string): FilePath {
|
56
|
+
if (getFeatureFlag('cachePerformanceImprovements')) {
|
57
|
+
const cleanId = hashString(cacheId);
|
58
|
+
return path.join(this.dir, cleanId.slice(0, 2), cleanId.slice(2));
|
59
|
+
}
|
54
60
|
return path.join(this.dir, cacheId.slice(0, 2), cacheId.slice(2));
|
55
61
|
}
|
56
62
|
|
@@ -90,6 +96,9 @@ export class FSCache implements Cache {
|
|
90
96
|
}
|
91
97
|
|
92
98
|
#getFilePath(key: string, index: number): string {
|
99
|
+
if (getFeatureFlag('cachePerformanceImprovements')) {
|
100
|
+
return path.join(this.dir, `${hashString(key)}-${index}`);
|
101
|
+
}
|
93
102
|
return path.join(this.dir, `${key}-${index}`);
|
94
103
|
}
|
95
104
|
|
package/src/LMDBLiteCache.js
CHANGED
@@ -19,6 +19,7 @@ import {NodeFS} from '@atlaspack/fs';
|
|
19
19
|
// $FlowFixMe
|
20
20
|
import packageJson from '../package.json';
|
21
21
|
import {FSCache} from './FSCache';
|
22
|
+
import {instrumentAsync} from '@atlaspack/logger';
|
22
23
|
|
23
24
|
const ncpAsync = promisify(ncp);
|
24
25
|
|
@@ -35,11 +36,6 @@ export class LmdbWrapper {
|
|
35
36
|
|
36
37
|
constructor(lmdb: Lmdb) {
|
37
38
|
this.lmdb = lmdb;
|
38
|
-
|
39
|
-
// $FlowFixMe
|
40
|
-
this[Symbol.dispose] = () => {
|
41
|
-
this.lmdb.close();
|
42
|
-
};
|
43
39
|
}
|
44
40
|
|
45
41
|
has(key: string): boolean {
|
@@ -107,10 +103,15 @@ export class LMDBLiteCache implements Cache {
|
|
107
103
|
dir: FilePath;
|
108
104
|
store: LmdbWrapper;
|
109
105
|
fsCache: FSCache;
|
106
|
+
/**
|
107
|
+
* Directory where we store raw files.
|
108
|
+
*/
|
109
|
+
cacheFilesDirectory: FilePath;
|
110
110
|
|
111
111
|
constructor(cacheDir: FilePath) {
|
112
112
|
this.fs = new NodeFS();
|
113
113
|
this.dir = cacheDir;
|
114
|
+
this.cacheFilesDirectory = path.join(cacheDir, 'files');
|
114
115
|
this.fsCache = new FSCache(this.fs, cacheDir);
|
115
116
|
|
116
117
|
this.store = open(cacheDir, {
|
@@ -131,6 +132,7 @@ export class LMDBLiteCache implements Cache {
|
|
131
132
|
if (!getFeatureFlag('cachePerformanceImprovements')) {
|
132
133
|
await this.fsCache.ensure();
|
133
134
|
}
|
135
|
+
await this.fs.mkdirp(this.cacheFilesDirectory);
|
134
136
|
return Promise.resolve();
|
135
137
|
}
|
136
138
|
|
@@ -162,14 +164,24 @@ export class LMDBLiteCache implements Cache {
|
|
162
164
|
}
|
163
165
|
|
164
166
|
getStream(key: string): Readable {
|
165
|
-
|
167
|
+
if (!getFeatureFlag('cachePerformanceImprovements')) {
|
168
|
+
return this.fs.createReadStream(path.join(this.dir, key));
|
169
|
+
}
|
170
|
+
|
171
|
+
return fs.createReadStream(this.getFileKey(key));
|
166
172
|
}
|
167
173
|
|
168
|
-
setStream(key: string, stream: Readable): Promise<void> {
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
174
|
+
async setStream(key: string, stream: Readable): Promise<void> {
|
175
|
+
if (!getFeatureFlag('cachePerformanceImprovements')) {
|
176
|
+
return pipeline(
|
177
|
+
stream,
|
178
|
+
this.fs.createWriteStream(path.join(this.dir, key)),
|
179
|
+
);
|
180
|
+
}
|
181
|
+
|
182
|
+
const filePath = this.getFileKey(key);
|
183
|
+
await fs.promises.mkdir(path.dirname(filePath), {recursive: true});
|
184
|
+
return pipeline(stream, fs.createWriteStream(filePath));
|
173
185
|
}
|
174
186
|
|
175
187
|
// eslint-disable-next-line require-await
|
@@ -193,31 +205,25 @@ export class LMDBLiteCache implements Cache {
|
|
193
205
|
return Promise.resolve(this.store.get(key));
|
194
206
|
}
|
195
207
|
|
196
|
-
#getFilePath(key: string, index: number): string {
|
197
|
-
return path.join(this.dir, `${key}-${index}`);
|
198
|
-
}
|
199
|
-
|
200
208
|
hasLargeBlob(key: string): Promise<boolean> {
|
201
209
|
if (!getFeatureFlag('cachePerformanceImprovements')) {
|
202
210
|
return this.fsCache.hasLargeBlob(key);
|
203
211
|
}
|
204
|
-
|
212
|
+
|
213
|
+
return fs.promises
|
214
|
+
.access(this.getFileKey(key), fs.constants.F_OK)
|
215
|
+
.then(() => true)
|
216
|
+
.catch(() => false);
|
205
217
|
}
|
206
218
|
|
207
|
-
/**
|
208
|
-
* @deprecated Use getBlob instead.
|
209
|
-
*/
|
210
219
|
getLargeBlob(key: string): Promise<Buffer> {
|
211
220
|
if (!getFeatureFlag('cachePerformanceImprovements')) {
|
212
221
|
return this.fsCache.getLargeBlob(key);
|
213
222
|
}
|
214
|
-
return
|
223
|
+
return fs.promises.readFile(this.getFileKey(key));
|
215
224
|
}
|
216
225
|
|
217
|
-
|
218
|
-
* @deprecated Use setBlob instead.
|
219
|
-
*/
|
220
|
-
setLargeBlob(
|
226
|
+
async setLargeBlob(
|
221
227
|
key: string,
|
222
228
|
contents: Buffer | string,
|
223
229
|
options?: {|signal?: AbortSignal|},
|
@@ -225,7 +231,10 @@ export class LMDBLiteCache implements Cache {
|
|
225
231
|
if (!getFeatureFlag('cachePerformanceImprovements')) {
|
226
232
|
return this.fsCache.setLargeBlob(key, contents, options);
|
227
233
|
}
|
228
|
-
|
234
|
+
|
235
|
+
const targetPath = this.getFileKey(key);
|
236
|
+
await fs.promises.mkdir(path.dirname(targetPath), {recursive: true});
|
237
|
+
return fs.promises.writeFile(targetPath, contents);
|
229
238
|
}
|
230
239
|
|
231
240
|
/**
|
@@ -262,6 +271,42 @@ export class LMDBLiteCache implements Cache {
|
|
262
271
|
}
|
263
272
|
|
264
273
|
refresh(): void {}
|
274
|
+
|
275
|
+
/**
|
276
|
+
* Streams, packages are stored in files instead of LMDB.
|
277
|
+
*
|
278
|
+
* On this case, if a cache key happens to have a parent traversal, ../..
|
279
|
+
* it is treated specially
|
280
|
+
*
|
281
|
+
* That is, something/../something and something are meant to be different
|
282
|
+
* keys.
|
283
|
+
*
|
284
|
+
* Plus we do not want to store values outside of the cache directory.
|
285
|
+
*/
|
286
|
+
getFileKey(key: string): string {
|
287
|
+
const cleanKey = key
|
288
|
+
.split('/')
|
289
|
+
.map((part) => {
|
290
|
+
if (part === '..') {
|
291
|
+
return '$$__parent_dir$$';
|
292
|
+
}
|
293
|
+
return part;
|
294
|
+
})
|
295
|
+
.join('/');
|
296
|
+
return path.join(this.cacheFilesDirectory, cleanKey);
|
297
|
+
}
|
298
|
+
|
299
|
+
async clear(): Promise<void> {
|
300
|
+
await instrumentAsync('LMDBLiteCache::clear', async () => {
|
301
|
+
const keys = await this.keys();
|
302
|
+
for (const key of keys) {
|
303
|
+
await this.store.delete(key);
|
304
|
+
}
|
305
|
+
|
306
|
+
await this.fs.rimraf(this.cacheFilesDirectory);
|
307
|
+
await this.fs.mkdirp(this.cacheFilesDirectory);
|
308
|
+
});
|
309
|
+
}
|
265
310
|
}
|
266
311
|
|
267
312
|
registerSerializableClass(
|
@@ -6,6 +6,8 @@ import {tmpdir} from 'os';
|
|
6
6
|
import {LMDBLiteCache} from '../src/index';
|
7
7
|
import {deserialize, serialize} from 'v8';
|
8
8
|
import assert from 'assert';
|
9
|
+
import {Worker} from 'worker_threads';
|
10
|
+
import {initializeMonitoring} from '@atlaspack/rust';
|
9
11
|
|
10
12
|
const cacheDir = path.join(tmpdir(), 'lmdb-lite-cache-tests');
|
11
13
|
|
@@ -16,10 +18,6 @@ describe('LMDBLiteCache', () => {
|
|
16
18
|
await fs.promises.rm(cacheDir, {recursive: true, force: true});
|
17
19
|
});
|
18
20
|
|
19
|
-
afterEach(() => {
|
20
|
-
cache.getNativeRef().close();
|
21
|
-
});
|
22
|
-
|
23
21
|
it('can be constructed', async () => {
|
24
22
|
cache = new LMDBLiteCache(cacheDir);
|
25
23
|
await cache.ensure();
|
@@ -59,11 +57,187 @@ describe('LMDBLiteCache', () => {
|
|
59
57
|
await cache.setBlob('key2', Buffer.from(serialize({value: 43})));
|
60
58
|
await cache.compact(path.join(cacheDir, 'compact_test_compacted'));
|
61
59
|
|
62
|
-
cache.getNativeRef().close();
|
63
|
-
|
64
60
|
cache = new LMDBLiteCache(path.join(cacheDir, 'compact_test_compacted'));
|
65
61
|
await cache.ensure();
|
66
62
|
const keys = cache.keys();
|
67
63
|
assert.deepEqual(Array.from(keys), ['key1', 'key2']);
|
68
64
|
});
|
65
|
+
|
66
|
+
describe('getFileKey', () => {
|
67
|
+
it('should return the correct key', () => {
|
68
|
+
const target = path.join(cacheDir, 'test-file-keys');
|
69
|
+
const cache = new LMDBLiteCache(target);
|
70
|
+
const key = cache.getFileKey('key');
|
71
|
+
assert.equal(key, path.join(target, 'files', 'key'));
|
72
|
+
});
|
73
|
+
|
74
|
+
it('should return the correct key for a key with a parent traversal', () => {
|
75
|
+
const target = path.join(cacheDir, 'test-parent-keys');
|
76
|
+
cache = new LMDBLiteCache(target);
|
77
|
+
const key = cache.getFileKey('../../key');
|
78
|
+
assert.equal(
|
79
|
+
key,
|
80
|
+
path.join(target, 'files', '$$__parent_dir$$/$$__parent_dir$$/key'),
|
81
|
+
);
|
82
|
+
});
|
83
|
+
});
|
84
|
+
|
85
|
+
it('can be closed and re-opened', async () => {
|
86
|
+
cache = new LMDBLiteCache(path.join(cacheDir, 'close_and_reopen_test'));
|
87
|
+
await cache.ensure();
|
88
|
+
await cache.setBlob('key', Buffer.from(serialize({value: 42})));
|
89
|
+
cache = new LMDBLiteCache(path.join(cacheDir, 'close_and_reopen_test'));
|
90
|
+
await cache.ensure();
|
91
|
+
const buffer = await cache.getBlob('key');
|
92
|
+
const result = deserialize(buffer);
|
93
|
+
assert.equal(result.value, 42);
|
94
|
+
});
|
95
|
+
|
96
|
+
it('should NOT fail when trying to open the same database twice', async () => {
|
97
|
+
const testDir = path.join(cacheDir, 'double_open_test');
|
98
|
+
const cache1 = new LMDBLiteCache(testDir);
|
99
|
+
await cache1.ensure();
|
100
|
+
|
101
|
+
assert.doesNotThrow(() => {
|
102
|
+
new LMDBLiteCache(testDir);
|
103
|
+
});
|
104
|
+
});
|
105
|
+
|
106
|
+
it('should NOT fail when trying to open after GC', async () => {
|
107
|
+
const testDir = path.join(cacheDir, 'gc_test');
|
108
|
+
|
109
|
+
let cache1 = new LMDBLiteCache(testDir);
|
110
|
+
await cache1.ensure();
|
111
|
+
await cache1.setBlob('key', Buffer.from(serialize({value: 42})));
|
112
|
+
|
113
|
+
cache1 = null;
|
114
|
+
|
115
|
+
if (global.gc) {
|
116
|
+
global.gc();
|
117
|
+
}
|
118
|
+
|
119
|
+
assert.doesNotThrow(() => {
|
120
|
+
new LMDBLiteCache(testDir);
|
121
|
+
});
|
122
|
+
});
|
123
|
+
|
124
|
+
it('should handle rapid open/close cycles', async () => {
|
125
|
+
const testDir = path.join(cacheDir, 'rapid_cycles_test');
|
126
|
+
|
127
|
+
for (let i = 0; i < 10; i++) {
|
128
|
+
const cache = new LMDBLiteCache(testDir);
|
129
|
+
await cache.ensure();
|
130
|
+
await cache.setBlob(`key${i}`, Buffer.from(serialize({value: i})));
|
131
|
+
|
132
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
133
|
+
}
|
134
|
+
|
135
|
+
const finalCache = new LMDBLiteCache(testDir);
|
136
|
+
await finalCache.ensure();
|
137
|
+
const buffer = await finalCache.getBlob('key9');
|
138
|
+
const result = deserialize(buffer);
|
139
|
+
assert.equal(result.value, 9);
|
140
|
+
});
|
141
|
+
|
142
|
+
it('should work when there are multiple node.js worker threads accessing the same database', async function () {
|
143
|
+
this.timeout(40000);
|
144
|
+
|
145
|
+
try {
|
146
|
+
initializeMonitoring();
|
147
|
+
} catch (error) {
|
148
|
+
/* empty */
|
149
|
+
}
|
150
|
+
|
151
|
+
const testDir = path.join(cacheDir, 'worker_threads_test');
|
152
|
+
|
153
|
+
let cache = new LMDBLiteCache(testDir);
|
154
|
+
await cache.set('main_thread_key', {
|
155
|
+
mainThreadId: 0,
|
156
|
+
hello: 'world',
|
157
|
+
});
|
158
|
+
setTimeout(() => {
|
159
|
+
cache = null;
|
160
|
+
|
161
|
+
if (global.gc) {
|
162
|
+
global.gc();
|
163
|
+
}
|
164
|
+
}, Math.random() * 300);
|
165
|
+
|
166
|
+
const numWorkers = 10;
|
167
|
+
|
168
|
+
const workers = [];
|
169
|
+
const responsePromises = [];
|
170
|
+
for (let i = 0; i < numWorkers; i++) {
|
171
|
+
const worker = new Worker(path.join(__dirname, 'workerThreadsTest.js'), {
|
172
|
+
workerData: {
|
173
|
+
cacheDir: testDir,
|
174
|
+
},
|
175
|
+
});
|
176
|
+
workers.push(worker);
|
177
|
+
|
178
|
+
const responsePromise = new Promise((resolve, reject) => {
|
179
|
+
worker.addListener('error', (error: Error) => {
|
180
|
+
reject(error);
|
181
|
+
});
|
182
|
+
worker.addListener('message', (message) => {
|
183
|
+
resolve(message);
|
184
|
+
});
|
185
|
+
});
|
186
|
+
|
187
|
+
worker.addListener('message', (message) => {
|
188
|
+
// eslint-disable-next-line no-console
|
189
|
+
console.log('Worker message', message);
|
190
|
+
});
|
191
|
+
worker.addListener('online', () => {
|
192
|
+
worker.postMessage({
|
193
|
+
type: 'go',
|
194
|
+
});
|
195
|
+
});
|
196
|
+
|
197
|
+
responsePromises.push(responsePromise);
|
198
|
+
}
|
199
|
+
|
200
|
+
// eslint-disable-next-line no-console
|
201
|
+
console.log('Waiting for responses');
|
202
|
+
const responses = await Promise.all(responsePromises);
|
203
|
+
|
204
|
+
// eslint-disable-next-line no-console
|
205
|
+
console.log('Responses received');
|
206
|
+
for (const [index, response] of responses.entries()) {
|
207
|
+
const worker = workers[index];
|
208
|
+
|
209
|
+
assert.deepEqual(
|
210
|
+
response,
|
211
|
+
{
|
212
|
+
mainThreadData: {
|
213
|
+
mainThreadId: 0,
|
214
|
+
hello: 'world',
|
215
|
+
},
|
216
|
+
workerId: worker.threadId,
|
217
|
+
},
|
218
|
+
`worker_${index} - Worker ${worker.threadId} should have received the correct data`,
|
219
|
+
);
|
220
|
+
}
|
221
|
+
|
222
|
+
// eslint-disable-next-line no-console
|
223
|
+
console.log('Getting main thread key');
|
224
|
+
cache = new LMDBLiteCache(testDir);
|
225
|
+
const data = await cache?.get('main_thread_key');
|
226
|
+
assert.deepEqual(data, {
|
227
|
+
mainThreadId: 0,
|
228
|
+
hello: 'world',
|
229
|
+
});
|
230
|
+
|
231
|
+
// eslint-disable-next-line no-console
|
232
|
+
console.log('Getting worker keys');
|
233
|
+
for (const worker of workers) {
|
234
|
+
const data = await cache?.get(`worker_key/${worker.threadId}`);
|
235
|
+
assert.deepEqual(data, {
|
236
|
+
workerId: worker.threadId,
|
237
|
+
});
|
238
|
+
|
239
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
240
|
+
worker.terminate();
|
241
|
+
}
|
242
|
+
});
|
69
243
|
});
|
@@ -0,0 +1,42 @@
|
|
1
|
+
/* eslint-disable no-inner-declarations */
|
2
|
+
|
3
|
+
require('@atlaspack/babel-register');
|
4
|
+
const {
|
5
|
+
workerData,
|
6
|
+
threadId,
|
7
|
+
parentPort,
|
8
|
+
isMainThread,
|
9
|
+
} = require('worker_threads');
|
10
|
+
const {LMDBLiteCache} = require('../src/index');
|
11
|
+
|
12
|
+
if (!isMainThread) {
|
13
|
+
const cache = new LMDBLiteCache(workerData.cacheDir);
|
14
|
+
|
15
|
+
async function onMessage() {
|
16
|
+
try {
|
17
|
+
cache.set(`worker_key/${threadId}`, {
|
18
|
+
workerId: threadId,
|
19
|
+
});
|
20
|
+
|
21
|
+
const data = await cache.get('main_thread_key');
|
22
|
+
|
23
|
+
parentPort.postMessage({
|
24
|
+
mainThreadData: data,
|
25
|
+
workerId: threadId,
|
26
|
+
});
|
27
|
+
|
28
|
+
setTimeout(() => {
|
29
|
+
parentPort.postMessage({
|
30
|
+
type: 'close',
|
31
|
+
workerId: threadId,
|
32
|
+
});
|
33
|
+
}, Math.random() * 200);
|
34
|
+
} catch (error) {
|
35
|
+
parentPort.postMessage({
|
36
|
+
error: error.message,
|
37
|
+
});
|
38
|
+
}
|
39
|
+
}
|
40
|
+
|
41
|
+
parentPort.on('message', onMessage);
|
42
|
+
}
|