@atlaspack/cache 2.13.3-dev.144 → 2.13.3-dev.146

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,114 @@
1
1
  # @atlaspack/cache
2
2
 
3
+ ## 3.2.5
4
+
5
+ ### Patch Changes
6
+
7
+ - Updated dependencies [[`0999fb7`](https://github.com/atlassian-labs/atlaspack/commit/0999fb78da519a6c7582d212883e515fcf6c1252), [`35fdd4b`](https://github.com/atlassian-labs/atlaspack/commit/35fdd4b52da0af20f74667f7b8adfb2f90279b7c), [`6dd4ccb`](https://github.com/atlassian-labs/atlaspack/commit/6dd4ccb753541de32322d881f973d571dd57e4ca)]:
8
+ - @atlaspack/fs@2.15.5
9
+ - @atlaspack/rust@3.3.5
10
+ - @atlaspack/logger@2.14.10
11
+ - @atlaspack/utils@2.14.10
12
+
13
+ ## 3.2.4
14
+
15
+ ### Patch Changes
16
+
17
+ - [#583](https://github.com/atlassian-labs/atlaspack/pull/583) [`124b7ff`](https://github.com/atlassian-labs/atlaspack/commit/124b7fff44f71aac9fbad289a9a9509b3dfc9aaa) Thanks [@yamadapc](https://github.com/yamadapc)! - Fix problem where cache writes could start to fail during a V3 build
18
+
19
+ - Updated dependencies [[`124b7ff`](https://github.com/atlassian-labs/atlaspack/commit/124b7fff44f71aac9fbad289a9a9509b3dfc9aaa), [`e052521`](https://github.com/atlassian-labs/atlaspack/commit/e0525210850ed1606146eb86991049cf567c5dec), [`15c6d70`](https://github.com/atlassian-labs/atlaspack/commit/15c6d7000bd89da876bc590aa75b17a619a41896), [`e4d966c`](https://github.com/atlassian-labs/atlaspack/commit/e4d966c3c9c4292c5013372ae65b10d19d4bacc6), [`209692f`](https://github.com/atlassian-labs/atlaspack/commit/209692ffb11eae103a0d65c5e1118a5aa1625818), [`42a775d`](https://github.com/atlassian-labs/atlaspack/commit/42a775de8eec638ad188f3271964170d8c04d84b), [`29c2f10`](https://github.com/atlassian-labs/atlaspack/commit/29c2f106de9679adfb5afa04e1910471dc65a427), [`f4da1e1`](https://github.com/atlassian-labs/atlaspack/commit/f4da1e120e73eeb5e8b8927f05e88f04d6148c7b), [`1ef91fc`](https://github.com/atlassian-labs/atlaspack/commit/1ef91fcc863fdd2831511937083dbbc1263b3d9d)]:
20
+ - @atlaspack/rust@3.3.4
21
+ - @atlaspack/fs@2.15.4
22
+ - @atlaspack/feature-flags@2.16.0
23
+ - @atlaspack/logger@2.14.9
24
+ - @atlaspack/utils@2.14.9
25
+
26
+ ## 3.2.3
27
+
28
+ ### Patch Changes
29
+
30
+ - Updated dependencies [[`30f6017`](https://github.com/atlassian-labs/atlaspack/commit/30f60175ba4d272c5fc193973c63bc298584775b), [`3a3e8e7`](https://github.com/atlassian-labs/atlaspack/commit/3a3e8e7be9e2dffd7304436d792f0f595d59665a), [`1ab0a27`](https://github.com/atlassian-labs/atlaspack/commit/1ab0a275aeca40350415e2b03e7440d1dddc6228), [`b8a4ae8`](https://github.com/atlassian-labs/atlaspack/commit/b8a4ae8f83dc0a83d8b145c5f729936ce52080a3)]:
31
+ - @atlaspack/feature-flags@2.15.1
32
+ - @atlaspack/fs@2.15.3
33
+ - @atlaspack/rust@3.3.3
34
+ - @atlaspack/utils@2.14.8
35
+ - @atlaspack/logger@2.14.8
36
+
37
+ ## 3.2.2
38
+
39
+ ### Patch Changes
40
+
41
+ - Updated dependencies [[`a1773d2`](https://github.com/atlassian-labs/atlaspack/commit/a1773d2a62d0ef7805ac7524621dcabcc1afe929), [`556d6ab`](https://github.com/atlassian-labs/atlaspack/commit/556d6ab8ede759fa7f37fcd3f4da336ef1c55e8f)]:
42
+ - @atlaspack/feature-flags@2.15.0
43
+ - @atlaspack/logger@2.14.7
44
+ - @atlaspack/rust@3.3.2
45
+ - @atlaspack/fs@2.15.2
46
+ - @atlaspack/utils@2.14.7
47
+
48
+ ## 3.2.1
49
+
50
+ ### Patch Changes
51
+
52
+ - Updated dependencies [[`e0f5337`](https://github.com/atlassian-labs/atlaspack/commit/e0f533757bd1019dbd108a04952c87da15286e09)]:
53
+ - @atlaspack/feature-flags@2.14.4
54
+ - @atlaspack/rust@3.3.1
55
+ - @atlaspack/fs@2.15.1
56
+ - @atlaspack/utils@2.14.6
57
+ - @atlaspack/logger@2.14.6
58
+
59
+ ## 3.2.0
60
+
61
+ ### Minor Changes
62
+
63
+ - [#531](https://github.com/atlassian-labs/atlaspack/pull/531) [`d2c50c2`](https://github.com/atlassian-labs/atlaspack/commit/d2c50c2c020888b33bb25b8690d9320c2b69e2a6) Thanks [@yamadapc](https://github.com/yamadapc)! - Add way to iterate LMDB cache keys
64
+
65
+ ### Patch Changes
66
+
67
+ - Updated dependencies [[`11d6f16`](https://github.com/atlassian-labs/atlaspack/commit/11d6f16b6397dee2f217167e5c98b39edb63f7a7), [`e2ba0f6`](https://github.com/atlassian-labs/atlaspack/commit/e2ba0f69702656f3d1ce95ab1454e35062b13b39), [`d2c50c2`](https://github.com/atlassian-labs/atlaspack/commit/d2c50c2c020888b33bb25b8690d9320c2b69e2a6), [`46a90dc`](https://github.com/atlassian-labs/atlaspack/commit/46a90dccd019a26b222c878a92d23acc75dc67c5), [`4c17141`](https://github.com/atlassian-labs/atlaspack/commit/4c1714103dab2aa9039c488f381551d2b65d1d01)]:
68
+ - @atlaspack/feature-flags@2.14.3
69
+ - @atlaspack/rust@3.3.0
70
+ - @atlaspack/fs@2.15.0
71
+ - @atlaspack/utils@2.14.5
72
+ - @atlaspack/logger@2.14.5
73
+
74
+ ## 3.1.0
75
+
76
+ ### Minor Changes
77
+
78
+ - [#525](https://github.com/atlassian-labs/atlaspack/pull/525) [`cb9da16`](https://github.com/atlassian-labs/atlaspack/commit/cb9da16fb2648e7f53c64df0313f60d5fb8970cc) Thanks [@yamadapc](https://github.com/yamadapc)! - Fix issues with large blob cache writes, run cache writes in a write transaction
79
+
80
+ ### Patch Changes
81
+
82
+ - Updated dependencies [[`1a2c14c`](https://github.com/atlassian-labs/atlaspack/commit/1a2c14c3cd4587551cc12e94d0680c8b71ea12bf), [`cb9da16`](https://github.com/atlassian-labs/atlaspack/commit/cb9da16fb2648e7f53c64df0313f60d5fb8970cc)]:
83
+ - @atlaspack/rust@3.2.0
84
+ - @atlaspack/fs@2.14.4
85
+ - @atlaspack/logger@2.14.4
86
+ - @atlaspack/utils@2.14.4
87
+
88
+ ## 3.0.1
89
+
90
+ ### Patch Changes
91
+
92
+ - Updated dependencies [[`f27d39e`](https://github.com/atlassian-labs/atlaspack/commit/f27d39e767b06def059944b3bc5fd50797eaea96)]:
93
+ - @atlaspack/rust@3.1.1
94
+ - @atlaspack/fs@2.14.3
95
+ - @atlaspack/logger@2.14.3
96
+ - @atlaspack/utils@2.14.3
97
+
98
+ ## 3.0.0
99
+
100
+ ### Major Changes
101
+
102
+ - [#512](https://github.com/atlassian-labs/atlaspack/pull/512) [`8f4e6c1`](https://github.com/atlassian-labs/atlaspack/commit/8f4e6c1b0e7c1fd48624afda48c1dcc599f1460f) Thanks [@yamadapc](https://github.com/yamadapc)! - Remove LMDB cache back-end
103
+
104
+ ### Patch Changes
105
+
106
+ - Updated dependencies [[`a891d65`](https://github.com/atlassian-labs/atlaspack/commit/a891d652bc4eb3d757d381adf65c5083f706effc), [`d02eab9`](https://github.com/atlassian-labs/atlaspack/commit/d02eab95eb60bf7457e0869af0b773608592c0e6), [`fb87a90`](https://github.com/atlassian-labs/atlaspack/commit/fb87a901973776b33ca4ce530e9d71669a9bd36d), [`7b9e8cf`](https://github.com/atlassian-labs/atlaspack/commit/7b9e8cf29e01a98e72e46b2b2fb74ccc514f4463)]:
107
+ - @atlaspack/rust@3.1.0
108
+ - @atlaspack/fs@2.14.2
109
+ - @atlaspack/utils@2.14.2
110
+ - @atlaspack/logger@2.14.2
111
+
3
112
  ## 2.13.3
4
113
 
5
114
  ### Patch Changes
package/index.d.ts CHANGED
@@ -7,10 +7,6 @@ export const FSCache: {
7
7
  new (cacheDir: FilePath): Cache;
8
8
  };
9
9
 
10
- export const LMDBCache: {
11
- new (cacheDir: FilePath): Cache;
12
- };
13
-
14
10
  export const LMDBLiteCache: {
15
11
  new (cacheDir: FilePath): Cache;
16
12
  };
package/lib/FSCache.js CHANGED
@@ -25,6 +25,20 @@ function _util() {
25
25
  };
26
26
  return data;
27
27
  }
28
+ function _rust() {
29
+ const data = require("@atlaspack/rust");
30
+ _rust = function () {
31
+ return data;
32
+ };
33
+ return data;
34
+ }
35
+ function _featureFlags() {
36
+ const data = require("@atlaspack/feature-flags");
37
+ _featureFlags = function () {
38
+ return data;
39
+ };
40
+ return data;
41
+ }
28
42
  function _logger() {
29
43
  const data = _interopRequireDefault(require("@atlaspack/logger"));
30
44
  _logger = function () {
@@ -62,6 +76,10 @@ class FSCache {
62
76
  await Promise.all(dirPromises);
63
77
  }
64
78
  _getCachePath(cacheId) {
79
+ if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
80
+ const cleanId = (0, _rust().hashString)(cacheId);
81
+ return _path().default.join(this.dir, cleanId.slice(0, 2), cleanId.slice(2));
82
+ }
65
83
  return _path().default.join(this.dir, cacheId.slice(0, 2), cacheId.slice(2));
66
84
  }
67
85
  getStream(key) {
@@ -91,6 +109,9 @@ class FSCache {
91
109
  }
92
110
  }
93
111
  #getFilePath(key, index) {
112
+ if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
113
+ return _path().default.join(this.dir, `${(0, _rust().hashString)(key)}-${index}`);
114
+ }
94
115
  return _path().default.join(this.dir, `${key}-${index}`);
95
116
  }
96
117
  async #unlinkChunks(key, index) {
@@ -12,6 +12,13 @@ function _buildCache() {
12
12
  };
13
13
  return data;
14
14
  }
15
+ function _featureFlags() {
16
+ const data = require("@atlaspack/feature-flags");
17
+ _featureFlags = function () {
18
+ return data;
19
+ };
20
+ return data;
21
+ }
15
22
  function _rust() {
16
23
  const data = require("@atlaspack/rust");
17
24
  _rust = function () {
@@ -19,16 +26,16 @@ function _rust() {
19
26
  };
20
27
  return data;
21
28
  }
22
- function _stream() {
23
- const data = _interopRequireDefault(require("stream"));
24
- _stream = function () {
29
+ function _fs() {
30
+ const data = _interopRequireDefault(require("fs"));
31
+ _fs = function () {
25
32
  return data;
26
33
  };
27
34
  return data;
28
35
  }
29
- function _path() {
30
- const data = _interopRequireDefault(require("path"));
31
- _path = function () {
36
+ function _ncp() {
37
+ const data = _interopRequireDefault(require("ncp"));
38
+ _ncp = function () {
32
39
  return data;
33
40
  };
34
41
  return data;
@@ -40,25 +47,48 @@ function _util() {
40
47
  };
41
48
  return data;
42
49
  }
43
- function _fs() {
50
+ function _stream() {
51
+ const data = _interopRequireDefault(require("stream"));
52
+ _stream = function () {
53
+ return data;
54
+ };
55
+ return data;
56
+ }
57
+ function _path() {
58
+ const data = _interopRequireDefault(require("path"));
59
+ _path = function () {
60
+ return data;
61
+ };
62
+ return data;
63
+ }
64
+ function _fs2() {
44
65
  const data = require("@atlaspack/fs");
45
- _fs = function () {
66
+ _fs2 = function () {
46
67
  return data;
47
68
  };
48
69
  return data;
49
70
  }
50
71
  var _package = _interopRequireDefault(require("../package.json"));
51
72
  var _FSCache = require("./FSCache");
73
+ function _logger() {
74
+ const data = require("@atlaspack/logger");
75
+ _logger = function () {
76
+ return data;
77
+ };
78
+ return data;
79
+ }
52
80
  function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
53
81
  // $FlowFixMe
82
+ const ncpAsync = (0, _util().promisify)(_ncp().default);
54
83
  class LmdbWrapper {
55
84
  constructor(lmdb) {
56
85
  this.lmdb = lmdb;
57
-
58
- // $FlowFixMe
59
- this[Symbol.dispose] = () => {
60
- this.lmdb.close();
61
- };
86
+ }
87
+ has(key) {
88
+ return this.lmdb.hasSync(key);
89
+ }
90
+ async delete(key) {
91
+ await this.lmdb.delete(key);
62
92
  }
63
93
  get(key) {
64
94
  return this.lmdb.getSync(key);
@@ -67,7 +97,19 @@ class LmdbWrapper {
67
97
  const buffer = typeof value === 'string' ? Buffer.from(value) : value;
68
98
  await this.lmdb.put(key, buffer);
69
99
  }
70
- resetReadTxn() {}
100
+ *keys() {
101
+ const PAGE_SIZE = 10000000;
102
+ let currentKeys = this.lmdb.keysSync(0, PAGE_SIZE);
103
+ while (currentKeys.length > 0) {
104
+ for (const key of currentKeys) {
105
+ yield key;
106
+ }
107
+ currentKeys = this.lmdb.keysSync(currentKeys.length, PAGE_SIZE);
108
+ }
109
+ }
110
+ compact(targetPath) {
111
+ this.lmdb.compact(targetPath);
112
+ }
71
113
  }
72
114
  exports.LmdbWrapper = LmdbWrapper;
73
115
  function open(directory
@@ -81,9 +123,14 @@ function open(directory
81
123
  }
82
124
  const pipeline = (0, _util().promisify)(_stream().default.pipeline);
83
125
  class LMDBLiteCache {
126
+ /**
127
+ * Directory where we store raw files.
128
+ */
129
+
84
130
  constructor(cacheDir) {
85
- this.fs = new (_fs().NodeFS)();
131
+ this.fs = new (_fs2().NodeFS)();
86
132
  this.dir = cacheDir;
133
+ this.cacheFilesDirectory = _path().default.join(cacheDir, 'files');
87
134
  this.fsCache = new _FSCache.FSCache(this.fs, cacheDir);
88
135
  this.store = open(cacheDir, {
89
136
  name: 'parcel-cache',
@@ -99,7 +146,10 @@ class LMDBLiteCache {
99
146
  return this.store.lmdb;
100
147
  }
101
148
  async ensure() {
102
- await this.fsCache.ensure();
149
+ if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
150
+ await this.fsCache.ensure();
151
+ }
152
+ await this.fs.mkdirp(this.cacheFilesDirectory);
103
153
  return Promise.resolve();
104
154
  }
105
155
  serialize() {
@@ -111,7 +161,7 @@ class LMDBLiteCache {
111
161
  return new LMDBLiteCache(cache.dir);
112
162
  }
113
163
  has(key) {
114
- return Promise.resolve(this.store.get(key) != null);
164
+ return Promise.resolve(this.store.has(key));
115
165
  }
116
166
  get(key) {
117
167
  let data = this.store.get(key);
@@ -124,10 +174,20 @@ class LMDBLiteCache {
124
174
  await this.setBlob(key, (0, _buildCache().serialize)(value));
125
175
  }
126
176
  getStream(key) {
127
- return this.fs.createReadStream(_path().default.join(this.dir, key));
177
+ if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
178
+ return this.fs.createReadStream(_path().default.join(this.dir, key));
179
+ }
180
+ return _fs().default.createReadStream(this.getFileKey(key));
128
181
  }
129
- setStream(key, stream) {
130
- return pipeline(stream, this.fs.createWriteStream(_path().default.join(this.dir, key)));
182
+ async setStream(key, stream) {
183
+ if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
184
+ return pipeline(stream, this.fs.createWriteStream(_path().default.join(this.dir, key)));
185
+ }
186
+ const filePath = this.getFileKey(key);
187
+ await _fs().default.promises.mkdir(_path().default.dirname(filePath), {
188
+ recursive: true
189
+ });
190
+ return pipeline(stream, _fs().default.createWriteStream(filePath));
131
191
  }
132
192
 
133
193
  // eslint-disable-next-line require-await
@@ -147,31 +207,87 @@ class LMDBLiteCache {
147
207
  getBuffer(key) {
148
208
  return Promise.resolve(this.store.get(key));
149
209
  }
150
- #getFilePath(key, index) {
151
- return _path().default.join(this.dir, `${key}-${index}`);
152
- }
153
210
  hasLargeBlob(key) {
154
- return this.fs.exists(this.#getFilePath(key, 0));
211
+ if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
212
+ return this.fsCache.hasLargeBlob(key);
213
+ }
214
+ return _fs().default.promises.access(this.getFileKey(key), _fs().default.constants.F_OK).then(() => true).catch(() => false);
155
215
  }
156
-
157
- // eslint-disable-next-line require-await
158
- async getLargeBlob(key) {
159
- return this.fsCache.getLargeBlob(key);
216
+ getLargeBlob(key) {
217
+ if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
218
+ return this.fsCache.getLargeBlob(key);
219
+ }
220
+ return _fs().default.promises.readFile(this.getFileKey(key));
160
221
  }
161
-
162
- // eslint-disable-next-line require-await
163
222
  async setLargeBlob(key, contents, options) {
164
- return this.fsCache.setLargeBlob(key, contents, options);
223
+ if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
224
+ return this.fsCache.setLargeBlob(key, contents, options);
225
+ }
226
+ const targetPath = this.getFileKey(key);
227
+ await _fs().default.promises.mkdir(_path().default.dirname(targetPath), {
228
+ recursive: true
229
+ });
230
+ return _fs().default.promises.writeFile(targetPath, contents);
165
231
  }
232
+
233
+ /**
234
+ * @deprecated Use store.delete instead.
235
+ */
166
236
  deleteLargeBlob(key) {
167
- return this.fsCache.deleteLargeBlob(key);
168
- }
169
- refresh() {
170
- // Reset the read transaction for the store. This guarantees that
171
- // the next read will see the latest changes to the store.
172
- // Useful in scenarios where reads and writes are multi-threaded.
173
- // See https://github.com/kriszyp/lmdb-js#resetreadtxn-void
174
- this.store.resetReadTxn();
237
+ if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
238
+ return this.fsCache.deleteLargeBlob(key);
239
+ }
240
+ return this.store.delete(key);
241
+ }
242
+ keys() {
243
+ return this.store.keys();
244
+ }
245
+ async compact(targetPath) {
246
+ await _fs().default.promises.mkdir(targetPath, {
247
+ recursive: true
248
+ });
249
+ const files = await _fs().default.promises.readdir(this.dir);
250
+ // copy all files except data.mdb and lock.mdb to the target path (recursive)
251
+ for (const file of files) {
252
+ const filePath = _path().default.join(this.dir, file);
253
+ if (file === 'data.mdb' || file === 'lock.mdb') {
254
+ continue;
255
+ }
256
+ await ncpAsync(filePath, _path().default.join(targetPath, file));
257
+ }
258
+ this.store.compact(_path().default.join(targetPath, 'data.mdb'));
259
+ }
260
+ refresh() {}
261
+
262
+ /**
263
+ * Streams, packages are stored in files instead of LMDB.
264
+ *
265
+ * On this case, if a cache key happens to have a parent traversal, ../..
266
+ * it is treated specially
267
+ *
268
+ * That is, something/../something and something are meant to be different
269
+ * keys.
270
+ *
271
+ * Plus we do not want to store values outside of the cache directory.
272
+ */
273
+ getFileKey(key) {
274
+ const cleanKey = key.split('/').map(part => {
275
+ if (part === '..') {
276
+ return '$$__parent_dir$$';
277
+ }
278
+ return part;
279
+ }).join('/');
280
+ return _path().default.join(this.cacheFilesDirectory, cleanKey);
281
+ }
282
+ async clear() {
283
+ await (0, _logger().instrumentAsync)('LMDBLiteCache::clear', async () => {
284
+ const keys = await this.keys();
285
+ for (const key of keys) {
286
+ await this.store.delete(key);
287
+ }
288
+ await this.fs.rimraf(this.cacheFilesDirectory);
289
+ await this.fs.mkdirp(this.cacheFilesDirectory);
290
+ });
175
291
  }
176
292
  }
177
293
  exports.LMDBLiteCache = LMDBLiteCache;
package/lib/index.js CHANGED
@@ -25,17 +25,6 @@ Object.keys(_IDBCache).forEach(function (key) {
25
25
  }
26
26
  });
27
27
  });
28
- var _LMDBCache = require("./LMDBCache");
29
- Object.keys(_LMDBCache).forEach(function (key) {
30
- if (key === "default" || key === "__esModule") return;
31
- if (key in exports && exports[key] === _LMDBCache[key]) return;
32
- Object.defineProperty(exports, key, {
33
- enumerable: true,
34
- get: function () {
35
- return _LMDBCache[key];
36
- }
37
- });
38
- });
39
28
  var _LMDBLiteCache = require("./LMDBLiteCache");
40
29
  Object.keys(_LMDBLiteCache).forEach(function (key) {
41
30
  if (key === "default" || key === "__esModule") return;
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@atlaspack/cache",
3
3
  "description": "Interface for defining caches and file-system, IDB and LMDB implementations.",
4
- "version": "2.13.3-dev.144+ea7d58b70",
4
+ "version": "2.13.3-dev.146+33cba81e9",
5
5
  "license": "(MIT OR Apache-2.0)",
6
6
  "type": "commonjs",
7
7
  "publishConfig": {
@@ -23,19 +23,19 @@
23
23
  "check-ts": "tsc --noEmit index.d.ts"
24
24
  },
25
25
  "dependencies": {
26
- "@atlaspack/build-cache": "2.13.3-dev.144+ea7d58b70",
27
- "@atlaspack/fs": "2.14.1-dev.144+ea7d58b70",
28
- "@atlaspack/logger": "2.14.1-dev.144+ea7d58b70",
29
- "@atlaspack/rust": "3.0.1-dev.144+ea7d58b70",
30
- "@atlaspack/utils": "2.14.1-dev.144+ea7d58b70",
31
- "lmdb": "2.8.5"
26
+ "@atlaspack/build-cache": "2.13.3-dev.146+33cba81e9",
27
+ "@atlaspack/feature-flags": "2.14.1-dev.146+33cba81e9",
28
+ "@atlaspack/fs": "2.14.1-dev.146+33cba81e9",
29
+ "@atlaspack/logger": "2.14.1-dev.146+33cba81e9",
30
+ "@atlaspack/rust": "3.0.1-dev.146+33cba81e9",
31
+ "@atlaspack/utils": "2.14.1-dev.146+33cba81e9",
32
+ "ncp": "^2.0.0"
32
33
  },
33
34
  "devDependencies": {
34
35
  "idb": "^5.0.8"
35
36
  },
36
37
  "browser": {
37
- "./src/IDBCache.js": "./src/IDBCache.browser.js",
38
- "./src/LMDBCache.js": false
38
+ "./src/IDBCache.js": "./src/IDBCache.browser.js"
39
39
  },
40
- "gitHead": "ea7d58b70fbe4bbc6324d4053ac673ea307b7bf6"
40
+ "gitHead": "33cba81e98b6eb098014f5a19d7932821b95f075"
41
41
  }
package/src/FSCache.js CHANGED
@@ -9,6 +9,8 @@ import stream from 'stream';
9
9
  import path from 'path';
10
10
  import {promisify} from 'util';
11
11
 
12
+ import {hashString} from '@atlaspack/rust';
13
+ import {getFeatureFlag} from '@atlaspack/feature-flags';
12
14
  import logger from '@atlaspack/logger';
13
15
  import {
14
16
  deserialize,
@@ -51,6 +53,10 @@ export class FSCache implements Cache {
51
53
  }
52
54
 
53
55
  _getCachePath(cacheId: string): FilePath {
56
+ if (getFeatureFlag('cachePerformanceImprovements')) {
57
+ const cleanId = hashString(cacheId);
58
+ return path.join(this.dir, cleanId.slice(0, 2), cleanId.slice(2));
59
+ }
54
60
  return path.join(this.dir, cacheId.slice(0, 2), cacheId.slice(2));
55
61
  }
56
62
 
@@ -90,6 +96,9 @@ export class FSCache implements Cache {
90
96
  }
91
97
 
92
98
  #getFilePath(key: string, index: number): string {
99
+ if (getFeatureFlag('cachePerformanceImprovements')) {
100
+ return path.join(this.dir, `${hashString(key)}-${index}`);
101
+ }
93
102
  return path.join(this.dir, `${key}-${index}`);
94
103
  }
95
104
 
@@ -5,21 +5,23 @@ import {
5
5
  registerSerializableClass,
6
6
  serialize,
7
7
  } from '@atlaspack/build-cache';
8
+ import {getFeatureFlag} from '@atlaspack/feature-flags';
8
9
  import {Lmdb} from '@atlaspack/rust';
9
10
  import type {FilePath} from '@atlaspack/types';
10
11
  import type {Cache} from './types';
11
12
  import type {Readable, Writable} from 'stream';
12
-
13
+ import fs from 'fs';
14
+ import ncp from 'ncp';
15
+ import {promisify} from 'util';
13
16
  import stream from 'stream';
14
17
  import path from 'path';
15
- import {promisify} from 'util';
16
-
17
18
  import {NodeFS} from '@atlaspack/fs';
18
-
19
19
  // $FlowFixMe
20
20
  import packageJson from '../package.json';
21
-
22
21
  import {FSCache} from './FSCache';
22
+ import {instrumentAsync} from '@atlaspack/logger';
23
+
24
+ const ncpAsync = promisify(ncp);
23
25
 
24
26
  interface DBOpenOptions {
25
27
  name: string;
@@ -34,11 +36,14 @@ export class LmdbWrapper {
34
36
 
35
37
  constructor(lmdb: Lmdb) {
36
38
  this.lmdb = lmdb;
39
+ }
37
40
 
38
- // $FlowFixMe
39
- this[Symbol.dispose] = () => {
40
- this.lmdb.close();
41
- };
41
+ has(key: string): boolean {
42
+ return this.lmdb.hasSync(key);
43
+ }
44
+
45
+ async delete(key: string): Promise<void> {
46
+ await this.lmdb.delete(key);
42
47
  }
43
48
 
44
49
  get(key: string): Buffer | null {
@@ -51,7 +56,21 @@ export class LmdbWrapper {
51
56
  await this.lmdb.put(key, buffer);
52
57
  }
53
58
 
54
- resetReadTxn() {}
59
+ *keys(): Iterable<string> {
60
+ const PAGE_SIZE = 10000000;
61
+
62
+ let currentKeys = this.lmdb.keysSync(0, PAGE_SIZE);
63
+ while (currentKeys.length > 0) {
64
+ for (const key of currentKeys) {
65
+ yield key;
66
+ }
67
+ currentKeys = this.lmdb.keysSync(currentKeys.length, PAGE_SIZE);
68
+ }
69
+ }
70
+
71
+ compact(targetPath: string) {
72
+ this.lmdb.compact(targetPath);
73
+ }
55
74
  }
56
75
 
57
76
  export function open(
@@ -84,10 +103,15 @@ export class LMDBLiteCache implements Cache {
84
103
  dir: FilePath;
85
104
  store: LmdbWrapper;
86
105
  fsCache: FSCache;
106
+ /**
107
+ * Directory where we store raw files.
108
+ */
109
+ cacheFilesDirectory: FilePath;
87
110
 
88
111
  constructor(cacheDir: FilePath) {
89
112
  this.fs = new NodeFS();
90
113
  this.dir = cacheDir;
114
+ this.cacheFilesDirectory = path.join(cacheDir, 'files');
91
115
  this.fsCache = new FSCache(this.fs, cacheDir);
92
116
 
93
117
  this.store = open(cacheDir, {
@@ -105,7 +129,10 @@ export class LMDBLiteCache implements Cache {
105
129
  }
106
130
 
107
131
  async ensure(): Promise<void> {
108
- await this.fsCache.ensure();
132
+ if (!getFeatureFlag('cachePerformanceImprovements')) {
133
+ await this.fsCache.ensure();
134
+ }
135
+ await this.fs.mkdirp(this.cacheFilesDirectory);
109
136
  return Promise.resolve();
110
137
  }
111
138
 
@@ -120,7 +147,7 @@ export class LMDBLiteCache implements Cache {
120
147
  }
121
148
 
122
149
  has(key: string): Promise<boolean> {
123
- return Promise.resolve(this.store.get(key) != null);
150
+ return Promise.resolve(this.store.has(key));
124
151
  }
125
152
 
126
153
  get<T>(key: string): Promise<?T> {
@@ -137,14 +164,24 @@ export class LMDBLiteCache implements Cache {
137
164
  }
138
165
 
139
166
  getStream(key: string): Readable {
140
- return this.fs.createReadStream(path.join(this.dir, key));
167
+ if (!getFeatureFlag('cachePerformanceImprovements')) {
168
+ return this.fs.createReadStream(path.join(this.dir, key));
169
+ }
170
+
171
+ return fs.createReadStream(this.getFileKey(key));
141
172
  }
142
173
 
143
- setStream(key: string, stream: Readable): Promise<void> {
144
- return pipeline(
145
- stream,
146
- this.fs.createWriteStream(path.join(this.dir, key)),
147
- );
174
+ async setStream(key: string, stream: Readable): Promise<void> {
175
+ if (!getFeatureFlag('cachePerformanceImprovements')) {
176
+ return pipeline(
177
+ stream,
178
+ this.fs.createWriteStream(path.join(this.dir, key)),
179
+ );
180
+ }
181
+
182
+ const filePath = this.getFileKey(key);
183
+ await fs.promises.mkdir(path.dirname(filePath), {recursive: true});
184
+ return pipeline(stream, fs.createWriteStream(filePath));
148
185
  }
149
186
 
150
187
  // eslint-disable-next-line require-await
@@ -168,38 +205,107 @@ export class LMDBLiteCache implements Cache {
168
205
  return Promise.resolve(this.store.get(key));
169
206
  }
170
207
 
171
- #getFilePath(key: string, index: number): string {
172
- return path.join(this.dir, `${key}-${index}`);
173
- }
174
-
175
208
  hasLargeBlob(key: string): Promise<boolean> {
176
- return this.fs.exists(this.#getFilePath(key, 0));
209
+ if (!getFeatureFlag('cachePerformanceImprovements')) {
210
+ return this.fsCache.hasLargeBlob(key);
211
+ }
212
+
213
+ return fs.promises
214
+ .access(this.getFileKey(key), fs.constants.F_OK)
215
+ .then(() => true)
216
+ .catch(() => false);
177
217
  }
178
218
 
179
- // eslint-disable-next-line require-await
180
- async getLargeBlob(key: string): Promise<Buffer> {
181
- return this.fsCache.getLargeBlob(key);
219
+ getLargeBlob(key: string): Promise<Buffer> {
220
+ if (!getFeatureFlag('cachePerformanceImprovements')) {
221
+ return this.fsCache.getLargeBlob(key);
222
+ }
223
+ return fs.promises.readFile(this.getFileKey(key));
182
224
  }
183
225
 
184
- // eslint-disable-next-line require-await
185
226
  async setLargeBlob(
186
227
  key: string,
187
228
  contents: Buffer | string,
188
229
  options?: {|signal?: AbortSignal|},
189
230
  ): Promise<void> {
190
- return this.fsCache.setLargeBlob(key, contents, options);
231
+ if (!getFeatureFlag('cachePerformanceImprovements')) {
232
+ return this.fsCache.setLargeBlob(key, contents, options);
233
+ }
234
+
235
+ const targetPath = this.getFileKey(key);
236
+ await fs.promises.mkdir(path.dirname(targetPath), {recursive: true});
237
+ return fs.promises.writeFile(targetPath, contents);
191
238
  }
192
239
 
240
+ /**
241
+ * @deprecated Use store.delete instead.
242
+ */
193
243
  deleteLargeBlob(key: string): Promise<void> {
194
- return this.fsCache.deleteLargeBlob(key);
244
+ if (!getFeatureFlag('cachePerformanceImprovements')) {
245
+ return this.fsCache.deleteLargeBlob(key);
246
+ }
247
+
248
+ return this.store.delete(key);
249
+ }
250
+
251
+ keys(): Iterable<string> {
252
+ return this.store.keys();
253
+ }
254
+
255
+ async compact(targetPath: string): Promise<void> {
256
+ await fs.promises.mkdir(targetPath, {recursive: true});
257
+
258
+ const files = await fs.promises.readdir(this.dir);
259
+ // copy all files except data.mdb and lock.mdb to the target path (recursive)
260
+ for (const file of files) {
261
+ const filePath = path.join(this.dir, file);
262
+
263
+ if (file === 'data.mdb' || file === 'lock.mdb') {
264
+ continue;
265
+ }
266
+
267
+ await ncpAsync(filePath, path.join(targetPath, file));
268
+ }
269
+
270
+ this.store.compact(path.join(targetPath, 'data.mdb'));
195
271
  }
196
272
 
197
- refresh(): void {
198
- // Reset the read transaction for the store. This guarantees that
199
- // the next read will see the latest changes to the store.
200
- // Useful in scenarios where reads and writes are multi-threaded.
201
- // See https://github.com/kriszyp/lmdb-js#resetreadtxn-void
202
- this.store.resetReadTxn();
273
+ refresh(): void {}
274
+
275
+ /**
276
+ * Streams, packages are stored in files instead of LMDB.
277
+ *
278
+ * On this case, if a cache key happens to have a parent traversal, ../..
279
+ * it is treated specially
280
+ *
281
+ * That is, something/../something and something are meant to be different
282
+ * keys.
283
+ *
284
+ * Plus we do not want to store values outside of the cache directory.
285
+ */
286
+ getFileKey(key: string): string {
287
+ const cleanKey = key
288
+ .split('/')
289
+ .map((part) => {
290
+ if (part === '..') {
291
+ return '$$__parent_dir$$';
292
+ }
293
+ return part;
294
+ })
295
+ .join('/');
296
+ return path.join(this.cacheFilesDirectory, cleanKey);
297
+ }
298
+
299
+ async clear(): Promise<void> {
300
+ await instrumentAsync('LMDBLiteCache::clear', async () => {
301
+ const keys = await this.keys();
302
+ for (const key of keys) {
303
+ await this.store.delete(key);
304
+ }
305
+
306
+ await this.fs.rimraf(this.cacheFilesDirectory);
307
+ await this.fs.mkdirp(this.cacheFilesDirectory);
308
+ });
203
309
  }
204
310
  }
205
311
 
package/src/index.js CHANGED
@@ -2,7 +2,6 @@
2
2
 
3
3
  export * from './FSCache';
4
4
  export * from './IDBCache';
5
- export * from './LMDBCache';
6
5
  export * from './LMDBLiteCache';
7
6
 
8
7
  export type {Cache} from './types';
@@ -1,20 +1,30 @@
1
1
  // @flow
2
+
3
+ import * as fs from 'fs';
2
4
  import * as path from 'path';
3
5
  import {tmpdir} from 'os';
4
6
  import {LMDBLiteCache} from '../src/index';
5
7
  import {deserialize, serialize} from 'v8';
6
8
  import assert from 'assert';
9
+ import {Worker} from 'worker_threads';
10
+ import {initializeMonitoring} from '@atlaspack/rust';
7
11
 
8
12
  const cacheDir = path.join(tmpdir(), 'lmdb-lite-cache-tests');
9
13
 
10
14
  describe('LMDBLiteCache', () => {
15
+ let cache;
16
+
17
+ beforeEach(async () => {
18
+ await fs.promises.rm(cacheDir, {recursive: true, force: true});
19
+ });
20
+
11
21
  it('can be constructed', async () => {
12
- const cache = new LMDBLiteCache(cacheDir);
22
+ cache = new LMDBLiteCache(cacheDir);
13
23
  await cache.ensure();
14
24
  });
15
25
 
16
26
  it('can retrieve keys', async () => {
17
- const cache = new LMDBLiteCache(cacheDir);
27
+ cache = new LMDBLiteCache(cacheDir);
18
28
  await cache.ensure();
19
29
  await cache.setBlob('key', Buffer.from(serialize({value: 42})));
20
30
  const buffer = await cache.getBlob('key');
@@ -23,11 +33,211 @@ describe('LMDBLiteCache', () => {
23
33
  });
24
34
 
25
35
  it('can retrieve keys synchronously', async () => {
26
- const cache = new LMDBLiteCache(cacheDir);
36
+ cache = new LMDBLiteCache(path.join(cacheDir, 'retrieve_keys_test'));
27
37
  await cache.ensure();
28
- cache.setBlob('key', Buffer.from(serialize({value: 42})));
38
+ await cache.setBlob('key', Buffer.from(serialize({value: 42})));
29
39
  const buffer = cache.getBlobSync('key');
30
40
  const result = deserialize(buffer);
31
41
  assert.equal(result.value, 42);
32
42
  });
43
+
44
+ it('can iterate over keys', async () => {
45
+ cache = new LMDBLiteCache(path.join(cacheDir, 'keys_test'));
46
+ await cache.ensure();
47
+ await cache.setBlob('key1', Buffer.from(serialize({value: 42})));
48
+ await cache.setBlob('key2', Buffer.from(serialize({value: 43})));
49
+ const keys = cache.keys();
50
+ assert.deepEqual(Array.from(keys), ['key1', 'key2']);
51
+ });
52
+
53
+ it('can compact databases', async () => {
54
+ cache = new LMDBLiteCache(path.join(cacheDir, 'compact_test'));
55
+ await cache.ensure();
56
+ await cache.setBlob('key1', Buffer.from(serialize({value: 42})));
57
+ await cache.setBlob('key2', Buffer.from(serialize({value: 43})));
58
+ await cache.compact(path.join(cacheDir, 'compact_test_compacted'));
59
+
60
+ cache = new LMDBLiteCache(path.join(cacheDir, 'compact_test_compacted'));
61
+ await cache.ensure();
62
+ const keys = cache.keys();
63
+ assert.deepEqual(Array.from(keys), ['key1', 'key2']);
64
+ });
65
+
66
+ describe('getFileKey', () => {
67
+ it('should return the correct key', () => {
68
+ const target = path.join(cacheDir, 'test-file-keys');
69
+ const cache = new LMDBLiteCache(target);
70
+ const key = cache.getFileKey('key');
71
+ assert.equal(key, path.join(target, 'files', 'key'));
72
+ });
73
+
74
+ it('should return the correct key for a key with a parent traversal', () => {
75
+ const target = path.join(cacheDir, 'test-parent-keys');
76
+ cache = new LMDBLiteCache(target);
77
+ const key = cache.getFileKey('../../key');
78
+ assert.equal(
79
+ key,
80
+ path.join(target, 'files', '$$__parent_dir$$/$$__parent_dir$$/key'),
81
+ );
82
+ });
83
+ });
84
+
85
+ it('can be closed and re-opened', async () => {
86
+ cache = new LMDBLiteCache(path.join(cacheDir, 'close_and_reopen_test'));
87
+ await cache.ensure();
88
+ await cache.setBlob('key', Buffer.from(serialize({value: 42})));
89
+ cache = new LMDBLiteCache(path.join(cacheDir, 'close_and_reopen_test'));
90
+ await cache.ensure();
91
+ const buffer = await cache.getBlob('key');
92
+ const result = deserialize(buffer);
93
+ assert.equal(result.value, 42);
94
+ });
95
+
96
+ it('should NOT fail when trying to open the same database twice', async () => {
97
+ const testDir = path.join(cacheDir, 'double_open_test');
98
+ const cache1 = new LMDBLiteCache(testDir);
99
+ await cache1.ensure();
100
+
101
+ assert.doesNotThrow(() => {
102
+ new LMDBLiteCache(testDir);
103
+ });
104
+ });
105
+
106
+ it('should NOT fail when trying to open after GC', async () => {
107
+ const testDir = path.join(cacheDir, 'gc_test');
108
+
109
+ let cache1 = new LMDBLiteCache(testDir);
110
+ await cache1.ensure();
111
+ await cache1.setBlob('key', Buffer.from(serialize({value: 42})));
112
+
113
+ cache1 = null;
114
+
115
+ if (global.gc) {
116
+ global.gc();
117
+ }
118
+
119
+ assert.doesNotThrow(() => {
120
+ new LMDBLiteCache(testDir);
121
+ });
122
+ });
123
+
124
+ it('should handle rapid open/close cycles', async () => {
125
+ const testDir = path.join(cacheDir, 'rapid_cycles_test');
126
+
127
+ for (let i = 0; i < 10; i++) {
128
+ const cache = new LMDBLiteCache(testDir);
129
+ await cache.ensure();
130
+ await cache.setBlob(`key${i}`, Buffer.from(serialize({value: i})));
131
+
132
+ await new Promise((resolve) => setTimeout(resolve, 10));
133
+ }
134
+
135
+ const finalCache = new LMDBLiteCache(testDir);
136
+ await finalCache.ensure();
137
+ const buffer = await finalCache.getBlob('key9');
138
+ const result = deserialize(buffer);
139
+ assert.equal(result.value, 9);
140
+ });
141
+
142
+ it('should work when there are multiple node.js worker threads accessing the same database', async function () {
143
+ this.timeout(40000);
144
+
145
+ try {
146
+ initializeMonitoring();
147
+ } catch (error) {
148
+ /* empty */
149
+ }
150
+
151
+ const testDir = path.join(cacheDir, 'worker_threads_test');
152
+
153
+ let cache = new LMDBLiteCache(testDir);
154
+ await cache.set('main_thread_key', {
155
+ mainThreadId: 0,
156
+ hello: 'world',
157
+ });
158
+ setTimeout(() => {
159
+ cache = null;
160
+
161
+ if (global.gc) {
162
+ global.gc();
163
+ }
164
+ }, Math.random() * 300);
165
+
166
+ const numWorkers = 10;
167
+
168
+ const workers = [];
169
+ const responsePromises = [];
170
+ for (let i = 0; i < numWorkers; i++) {
171
+ const worker = new Worker(path.join(__dirname, 'workerThreadsTest.js'), {
172
+ workerData: {
173
+ cacheDir: testDir,
174
+ },
175
+ });
176
+ workers.push(worker);
177
+
178
+ const responsePromise = new Promise((resolve, reject) => {
179
+ worker.addListener('error', (error: Error) => {
180
+ reject(error);
181
+ });
182
+ worker.addListener('message', (message) => {
183
+ resolve(message);
184
+ });
185
+ });
186
+
187
+ worker.addListener('message', (message) => {
188
+ // eslint-disable-next-line no-console
189
+ console.log('Worker message', message);
190
+ });
191
+ worker.addListener('online', () => {
192
+ worker.postMessage({
193
+ type: 'go',
194
+ });
195
+ });
196
+
197
+ responsePromises.push(responsePromise);
198
+ }
199
+
200
+ // eslint-disable-next-line no-console
201
+ console.log('Waiting for responses');
202
+ const responses = await Promise.all(responsePromises);
203
+
204
+ // eslint-disable-next-line no-console
205
+ console.log('Responses received');
206
+ for (const [index, response] of responses.entries()) {
207
+ const worker = workers[index];
208
+
209
+ assert.deepEqual(
210
+ response,
211
+ {
212
+ mainThreadData: {
213
+ mainThreadId: 0,
214
+ hello: 'world',
215
+ },
216
+ workerId: worker.threadId,
217
+ },
218
+ `worker_${index} - Worker ${worker.threadId} should have received the correct data`,
219
+ );
220
+ }
221
+
222
+ // eslint-disable-next-line no-console
223
+ console.log('Getting main thread key');
224
+ cache = new LMDBLiteCache(testDir);
225
+ const data = await cache?.get('main_thread_key');
226
+ assert.deepEqual(data, {
227
+ mainThreadId: 0,
228
+ hello: 'world',
229
+ });
230
+
231
+ // eslint-disable-next-line no-console
232
+ console.log('Getting worker keys');
233
+ for (const worker of workers) {
234
+ const data = await cache?.get(`worker_key/${worker.threadId}`);
235
+ assert.deepEqual(data, {
236
+ workerId: worker.threadId,
237
+ });
238
+
239
+ await new Promise((resolve) => setTimeout(resolve, 500));
240
+ worker.terminate();
241
+ }
242
+ });
33
243
  });
@@ -0,0 +1,42 @@
1
+ /* eslint-disable no-inner-declarations */
2
+
3
+ require('@atlaspack/babel-register');
4
+ const {
5
+ workerData,
6
+ threadId,
7
+ parentPort,
8
+ isMainThread,
9
+ } = require('worker_threads');
10
+ const {LMDBLiteCache} = require('../src/index');
11
+
12
+ if (!isMainThread) {
13
+ const cache = new LMDBLiteCache(workerData.cacheDir);
14
+
15
+ async function onMessage() {
16
+ try {
17
+ cache.set(`worker_key/${threadId}`, {
18
+ workerId: threadId,
19
+ });
20
+
21
+ const data = await cache.get('main_thread_key');
22
+
23
+ parentPort.postMessage({
24
+ mainThreadData: data,
25
+ workerId: threadId,
26
+ });
27
+
28
+ setTimeout(() => {
29
+ parentPort.postMessage({
30
+ type: 'close',
31
+ workerId: threadId,
32
+ });
33
+ }, Math.random() * 200);
34
+ } catch (error) {
35
+ parentPort.postMessage({
36
+ error: error.message,
37
+ });
38
+ }
39
+ }
40
+
41
+ parentPort.on('message', onMessage);
42
+ }
package/lib/LMDBCache.js DELETED
@@ -1,136 +0,0 @@
1
- "use strict";
2
-
3
- Object.defineProperty(exports, "__esModule", {
4
- value: true
5
- });
6
- exports.LMDBCache = void 0;
7
- function _stream() {
8
- const data = _interopRequireDefault(require("stream"));
9
- _stream = function () {
10
- return data;
11
- };
12
- return data;
13
- }
14
- function _path() {
15
- const data = _interopRequireDefault(require("path"));
16
- _path = function () {
17
- return data;
18
- };
19
- return data;
20
- }
21
- function _util() {
22
- const data = require("util");
23
- _util = function () {
24
- return data;
25
- };
26
- return data;
27
- }
28
- function _buildCache() {
29
- const data = require("@atlaspack/build-cache");
30
- _buildCache = function () {
31
- return data;
32
- };
33
- return data;
34
- }
35
- function _fs() {
36
- const data = require("@atlaspack/fs");
37
- _fs = function () {
38
- return data;
39
- };
40
- return data;
41
- }
42
- function _lmdb() {
43
- const data = _interopRequireDefault(require("lmdb"));
44
- _lmdb = function () {
45
- return data;
46
- };
47
- return data;
48
- }
49
- var _package = _interopRequireDefault(require("../package.json"));
50
- var _FSCache = require("./FSCache");
51
- function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
52
- // flowlint-next-line untyped-import:off
53
- // $FlowFixMe
54
- const pipeline = (0, _util().promisify)(_stream().default.pipeline);
55
- class LMDBCache {
56
- // $FlowFixMe
57
-
58
- constructor(cacheDir) {
59
- this.fs = new (_fs().NodeFS)();
60
- this.dir = cacheDir;
61
- this.fsCache = new _FSCache.FSCache(this.fs, cacheDir);
62
- this.store = _lmdb().default.open(cacheDir, {
63
- name: 'parcel-cache',
64
- encoding: 'binary',
65
- compression: true
66
- });
67
- }
68
- ensure() {
69
- return Promise.resolve();
70
- }
71
- serialize() {
72
- return {
73
- dir: this.dir
74
- };
75
- }
76
- static deserialize(opts) {
77
- return new LMDBCache(opts.dir);
78
- }
79
- has(key) {
80
- return Promise.resolve(this.store.get(key) != null);
81
- }
82
- get(key) {
83
- let data = this.store.get(key);
84
- if (data == null) {
85
- return Promise.resolve(null);
86
- }
87
- return Promise.resolve((0, _buildCache().deserialize)(data));
88
- }
89
- async set(key, value) {
90
- await this.setBlob(key, (0, _buildCache().serialize)(value));
91
- }
92
- getStream(key) {
93
- return this.fs.createReadStream(_path().default.join(this.dir, key));
94
- }
95
- setStream(key, stream) {
96
- return pipeline(stream, this.fs.createWriteStream(_path().default.join(this.dir, key)));
97
- }
98
- getBlob(key) {
99
- let buffer = this.store.get(key);
100
- return buffer != null ? Promise.resolve(buffer) : Promise.reject(new Error(`Key ${key} not found in cache`));
101
- }
102
- async setBlob(key, contents) {
103
- await this.store.put(key, contents);
104
- }
105
- getBuffer(key) {
106
- return Promise.resolve(this.store.get(key));
107
- }
108
- #getFilePath(key, index) {
109
- return _path().default.join(this.dir, `${key}-${index}`);
110
- }
111
- hasLargeBlob(key) {
112
- return this.fs.exists(this.#getFilePath(key, 0));
113
- }
114
-
115
- // eslint-disable-next-line require-await
116
- async getLargeBlob(key) {
117
- return this.fsCache.getLargeBlob(key);
118
- }
119
-
120
- // eslint-disable-next-line require-await
121
- async setLargeBlob(key, contents, options) {
122
- return this.fsCache.setLargeBlob(key, contents, options);
123
- }
124
- deleteLargeBlob(key) {
125
- return this.fsCache.deleteLargeBlob(key);
126
- }
127
- refresh() {
128
- // Reset the read transaction for the store. This guarantees that
129
- // the next read will see the latest changes to the store.
130
- // Useful in scenarios where reads and writes are multi-threaded.
131
- // See https://github.com/kriszyp/lmdb-js#resetreadtxn-void
132
- this.store.resetReadTxn();
133
- }
134
- }
135
- exports.LMDBCache = LMDBCache;
136
- (0, _buildCache().registerSerializableClass)(`${_package.default.version}:LMDBCache`, LMDBCache);
package/src/LMDBCache.js DELETED
@@ -1,139 +0,0 @@
1
- // @flow strict-local
2
- import type {FilePath} from '@atlaspack/types';
3
- import type {Cache} from './types';
4
- import type {Readable, Writable} from 'stream';
5
-
6
- import stream from 'stream';
7
- import path from 'path';
8
- import {promisify} from 'util';
9
-
10
- import {
11
- deserialize,
12
- registerSerializableClass,
13
- serialize,
14
- } from '@atlaspack/build-cache';
15
- import {NodeFS} from '@atlaspack/fs';
16
- // flowlint-next-line untyped-import:off
17
- import lmdb from 'lmdb';
18
-
19
- // $FlowFixMe
20
- import packageJson from '../package.json';
21
-
22
- import {FSCache} from './FSCache';
23
-
24
- const pipeline: (Readable, Writable) => Promise<void> = promisify(
25
- stream.pipeline,
26
- );
27
-
28
- export class LMDBCache implements Cache {
29
- fs: NodeFS;
30
- dir: FilePath;
31
- // $FlowFixMe
32
- store: any;
33
- fsCache: FSCache;
34
-
35
- constructor(cacheDir: FilePath) {
36
- this.fs = new NodeFS();
37
- this.dir = cacheDir;
38
- this.fsCache = new FSCache(this.fs, cacheDir);
39
-
40
- this.store = lmdb.open(cacheDir, {
41
- name: 'parcel-cache',
42
- encoding: 'binary',
43
- compression: true,
44
- });
45
- }
46
-
47
- ensure(): Promise<void> {
48
- return Promise.resolve();
49
- }
50
-
51
- serialize(): {|dir: FilePath|} {
52
- return {
53
- dir: this.dir,
54
- };
55
- }
56
-
57
- static deserialize(opts: {|dir: FilePath|}): LMDBCache {
58
- return new LMDBCache(opts.dir);
59
- }
60
-
61
- has(key: string): Promise<boolean> {
62
- return Promise.resolve(this.store.get(key) != null);
63
- }
64
-
65
- get<T>(key: string): Promise<?T> {
66
- let data = this.store.get(key);
67
- if (data == null) {
68
- return Promise.resolve(null);
69
- }
70
-
71
- return Promise.resolve(deserialize(data));
72
- }
73
-
74
- async set(key: string, value: mixed): Promise<void> {
75
- await this.setBlob(key, serialize(value));
76
- }
77
-
78
- getStream(key: string): Readable {
79
- return this.fs.createReadStream(path.join(this.dir, key));
80
- }
81
-
82
- setStream(key: string, stream: Readable): Promise<void> {
83
- return pipeline(
84
- stream,
85
- this.fs.createWriteStream(path.join(this.dir, key)),
86
- );
87
- }
88
-
89
- getBlob(key: string): Promise<Buffer> {
90
- let buffer = this.store.get(key);
91
- return buffer != null
92
- ? Promise.resolve(buffer)
93
- : Promise.reject(new Error(`Key ${key} not found in cache`));
94
- }
95
-
96
- async setBlob(key: string, contents: Buffer | string): Promise<void> {
97
- await this.store.put(key, contents);
98
- }
99
-
100
- getBuffer(key: string): Promise<?Buffer> {
101
- return Promise.resolve(this.store.get(key));
102
- }
103
-
104
- #getFilePath(key: string, index: number): string {
105
- return path.join(this.dir, `${key}-${index}`);
106
- }
107
-
108
- hasLargeBlob(key: string): Promise<boolean> {
109
- return this.fs.exists(this.#getFilePath(key, 0));
110
- }
111
-
112
- // eslint-disable-next-line require-await
113
- async getLargeBlob(key: string): Promise<Buffer> {
114
- return this.fsCache.getLargeBlob(key);
115
- }
116
-
117
- // eslint-disable-next-line require-await
118
- async setLargeBlob(
119
- key: string,
120
- contents: Buffer | string,
121
- options?: {|signal?: AbortSignal|},
122
- ): Promise<void> {
123
- return this.fsCache.setLargeBlob(key, contents, options);
124
- }
125
-
126
- deleteLargeBlob(key: string): Promise<void> {
127
- return this.fsCache.deleteLargeBlob(key);
128
- }
129
-
130
- refresh(): void {
131
- // Reset the read transaction for the store. This guarantees that
132
- // the next read will see the latest changes to the store.
133
- // Useful in scenarios where reads and writes are multi-threaded.
134
- // See https://github.com/kriszyp/lmdb-js#resetreadtxn-void
135
- this.store.resetReadTxn();
136
- }
137
- }
138
-
139
- registerSerializableClass(`${packageJson.version}:LMDBCache`, LMDBCache);