@atlaspack/cache 3.1.1-canary.0 → 3.1.1-canary.100

package/CHANGELOG.md

@@ -1,5 +1,122 @@
 # @atlaspack/cache
 
+## 3.2.10
+
+### Patch Changes
+
+- Updated dependencies [[`5ded263`](https://github.com/atlassian-labs/atlaspack/commit/5ded263c7f11b866e8885b81c73e20dd060b25be)]:
+  - @atlaspack/feature-flags@2.18.3
+  - @atlaspack/fs@2.15.10
+  - @atlaspack/utils@2.15.3
+
+## 3.2.9
+
+### Patch Changes
+
+- Updated dependencies [[`644b157`](https://github.com/atlassian-labs/atlaspack/commit/644b157dee72a871acc2d0facf0b87b8eea51956)]:
+  - @atlaspack/feature-flags@2.18.2
+  - @atlaspack/fs@2.15.9
+  - @atlaspack/utils@2.15.2
+
+## 3.2.8
+
+### Patch Changes
+
+- Updated dependencies [[`ef3d622`](https://github.com/atlassian-labs/atlaspack/commit/ef3d6228f4e006702198a19c61e051d194d325cb), [`26aa9c5`](https://github.com/atlassian-labs/atlaspack/commit/26aa9c599d2be45ce1438a74c5fa22f39b9b554b), [`0501255`](https://github.com/atlassian-labs/atlaspack/commit/05012550da35b05ce7d356a8cc29311e7f9afdca)]:
+  - @atlaspack/logger@2.14.11
+  - @atlaspack/feature-flags@2.18.1
+  - @atlaspack/fs@2.15.8
+  - @atlaspack/utils@2.15.1
+
+## 3.2.7
+
+### Patch Changes
+
+- Updated dependencies [[`10fbcfb`](https://github.com/atlassian-labs/atlaspack/commit/10fbcfbfa49c7a83da5d7c40983e36e87f524a75), [`85c52d3`](https://github.com/atlassian-labs/atlaspack/commit/85c52d3f7717b3c84a118d18ab98cfbfd71dcbd2), [`e39c6cf`](https://github.com/atlassian-labs/atlaspack/commit/e39c6cf05f7e95ce5420dbcea66f401b1cbd397c)]:
+  - @atlaspack/feature-flags@2.18.0
+  - @atlaspack/utils@2.15.0
+  - @atlaspack/fs@2.15.7
+
+## 3.2.6
+
+### Patch Changes
+
+- Updated dependencies [[`73ea3c4`](https://github.com/atlassian-labs/atlaspack/commit/73ea3c4d85d4401fdd15abcbf988237e890e7ad3), [`b1b3693`](https://github.com/atlassian-labs/atlaspack/commit/b1b369317c66f8a431c170df2ebba4fa5b2e38ef)]:
+  - @atlaspack/feature-flags@2.17.0
+  - @atlaspack/fs@2.15.6
+  - @atlaspack/utils@2.14.11
+
+## 3.2.5
+
+### Patch Changes
+
+- Updated dependencies [[`0999fb7`](https://github.com/atlassian-labs/atlaspack/commit/0999fb78da519a6c7582d212883e515fcf6c1252), [`35fdd4b`](https://github.com/atlassian-labs/atlaspack/commit/35fdd4b52da0af20f74667f7b8adfb2f90279b7c), [`6dd4ccb`](https://github.com/atlassian-labs/atlaspack/commit/6dd4ccb753541de32322d881f973d571dd57e4ca)]:
+  - @atlaspack/fs@2.15.5
+  - @atlaspack/rust@3.3.5
+  - @atlaspack/logger@2.14.10
+  - @atlaspack/utils@2.14.10
+
+## 3.2.4
+
+### Patch Changes
+
+- [#583](https://github.com/atlassian-labs/atlaspack/pull/583) [`124b7ff`](https://github.com/atlassian-labs/atlaspack/commit/124b7fff44f71aac9fbad289a9a9509b3dfc9aaa) Thanks [@yamadapc](https://github.com/yamadapc)! - Fix problem where cache writes could start to fail during a V3 build
+
+- Updated dependencies [[`124b7ff`](https://github.com/atlassian-labs/atlaspack/commit/124b7fff44f71aac9fbad289a9a9509b3dfc9aaa), [`e052521`](https://github.com/atlassian-labs/atlaspack/commit/e0525210850ed1606146eb86991049cf567c5dec), [`15c6d70`](https://github.com/atlassian-labs/atlaspack/commit/15c6d7000bd89da876bc590aa75b17a619a41896), [`e4d966c`](https://github.com/atlassian-labs/atlaspack/commit/e4d966c3c9c4292c5013372ae65b10d19d4bacc6), [`209692f`](https://github.com/atlassian-labs/atlaspack/commit/209692ffb11eae103a0d65c5e1118a5aa1625818), [`42a775d`](https://github.com/atlassian-labs/atlaspack/commit/42a775de8eec638ad188f3271964170d8c04d84b), [`29c2f10`](https://github.com/atlassian-labs/atlaspack/commit/29c2f106de9679adfb5afa04e1910471dc65a427), [`f4da1e1`](https://github.com/atlassian-labs/atlaspack/commit/f4da1e120e73eeb5e8b8927f05e88f04d6148c7b), [`1ef91fc`](https://github.com/atlassian-labs/atlaspack/commit/1ef91fcc863fdd2831511937083dbbc1263b3d9d)]:
+  - @atlaspack/rust@3.3.4
+  - @atlaspack/fs@2.15.4
+  - @atlaspack/feature-flags@2.16.0
+  - @atlaspack/logger@2.14.9
+  - @atlaspack/utils@2.14.9
+
+## 3.2.3
+
+### Patch Changes
+
+- Updated dependencies [[`30f6017`](https://github.com/atlassian-labs/atlaspack/commit/30f60175ba4d272c5fc193973c63bc298584775b), [`3a3e8e7`](https://github.com/atlassian-labs/atlaspack/commit/3a3e8e7be9e2dffd7304436d792f0f595d59665a), [`1ab0a27`](https://github.com/atlassian-labs/atlaspack/commit/1ab0a275aeca40350415e2b03e7440d1dddc6228), [`b8a4ae8`](https://github.com/atlassian-labs/atlaspack/commit/b8a4ae8f83dc0a83d8b145c5f729936ce52080a3)]:
+  - @atlaspack/feature-flags@2.15.1
+  - @atlaspack/fs@2.15.3
+  - @atlaspack/rust@3.3.3
+  - @atlaspack/utils@2.14.8
+  - @atlaspack/logger@2.14.8
+
+## 3.2.2
+
+### Patch Changes
+
+- Updated dependencies [[`a1773d2`](https://github.com/atlassian-labs/atlaspack/commit/a1773d2a62d0ef7805ac7524621dcabcc1afe929), [`556d6ab`](https://github.com/atlassian-labs/atlaspack/commit/556d6ab8ede759fa7f37fcd3f4da336ef1c55e8f)]:
+  - @atlaspack/feature-flags@2.15.0
+  - @atlaspack/logger@2.14.7
+  - @atlaspack/rust@3.3.2
+  - @atlaspack/fs@2.15.2
+  - @atlaspack/utils@2.14.7
+
+## 3.2.1
+
+### Patch Changes
+
+- Updated dependencies [[`e0f5337`](https://github.com/atlassian-labs/atlaspack/commit/e0f533757bd1019dbd108a04952c87da15286e09)]:
+  - @atlaspack/feature-flags@2.14.4
+  - @atlaspack/rust@3.3.1
+  - @atlaspack/fs@2.15.1
+  - @atlaspack/utils@2.14.6
+  - @atlaspack/logger@2.14.6
+
+## 3.2.0
+
+### Minor Changes
+
+- [#531](https://github.com/atlassian-labs/atlaspack/pull/531) [`d2c50c2`](https://github.com/atlassian-labs/atlaspack/commit/d2c50c2c020888b33bb25b8690d9320c2b69e2a6) Thanks [@yamadapc](https://github.com/yamadapc)! - Add way to iterate LMDB cache keys
+
+### Patch Changes
+
+- Updated dependencies [[`11d6f16`](https://github.com/atlassian-labs/atlaspack/commit/11d6f16b6397dee2f217167e5c98b39edb63f7a7), [`e2ba0f6`](https://github.com/atlassian-labs/atlaspack/commit/e2ba0f69702656f3d1ce95ab1454e35062b13b39), [`d2c50c2`](https://github.com/atlassian-labs/atlaspack/commit/d2c50c2c020888b33bb25b8690d9320c2b69e2a6), [`46a90dc`](https://github.com/atlassian-labs/atlaspack/commit/46a90dccd019a26b222c878a92d23acc75dc67c5), [`4c17141`](https://github.com/atlassian-labs/atlaspack/commit/4c1714103dab2aa9039c488f381551d2b65d1d01)]:
+  - @atlaspack/feature-flags@2.14.3
+  - @atlaspack/rust@3.3.0
+  - @atlaspack/fs@2.15.0
+  - @atlaspack/utils@2.14.5
+  - @atlaspack/logger@2.14.5
+
 ## 3.1.0
 
 ### Minor Changes

package/lib/FSCache.js

@@ -25,6 +25,20 @@ function _util() {
   };
   return data;
 }
+function _rust() {
+  const data = require("@atlaspack/rust");
+  _rust = function () {
+    return data;
+  };
+  return data;
+}
+function _featureFlags() {
+  const data = require("@atlaspack/feature-flags");
+  _featureFlags = function () {
+    return data;
+  };
+  return data;
+}
 function _logger() {
   const data = _interopRequireDefault(require("@atlaspack/logger"));
   _logger = function () {
@@ -62,6 +76,10 @@ class FSCache {
     await Promise.all(dirPromises);
   }
   _getCachePath(cacheId) {
+    if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
+      const cleanId = (0, _rust().hashString)(cacheId);
+      return _path().default.join(this.dir, cleanId.slice(0, 2), cleanId.slice(2));
+    }
     return _path().default.join(this.dir, cacheId.slice(0, 2), cacheId.slice(2));
   }
   getStream(key) {
@@ -91,6 +109,9 @@ class FSCache {
     }
   }
   #getFilePath(key, index) {
+    if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
+      return _path().default.join(this.dir, `${(0, _rust().hashString)(key)}-${index}`);
+    }
     return _path().default.join(this.dir, `${key}-${index}`);
   }
   async #unlinkChunks(key, index) {

package/lib/LMDBLiteCache.js

@@ -26,16 +26,16 @@ function _rust() {
   };
   return data;
 }
-function _stream() {
-  const data = _interopRequireDefault(require("stream"));
-  _stream = function () {
+function _fs() {
+  const data = _interopRequireDefault(require("fs"));
+  _fs = function () {
     return data;
   };
   return data;
 }
-function _path() {
-  const data = _interopRequireDefault(require("path"));
-  _path = function () {
+function _ncp() {
+  const data = _interopRequireDefault(require("ncp"));
+  _ncp = function () {
     return data;
   };
   return data;
@@ -47,25 +47,42 @@ function _util() {
   };
   return data;
 }
-function _fs() {
+function _stream() {
+  const data = _interopRequireDefault(require("stream"));
+  _stream = function () {
+    return data;
+  };
+  return data;
+}
+function _path() {
+  const data = _interopRequireDefault(require("path"));
+  _path = function () {
+    return data;
+  };
+  return data;
+}
+function _fs2() {
   const data = require("@atlaspack/fs");
-  _fs = function () {
+  _fs2 = function () {
     return data;
   };
   return data;
 }
 var _package = _interopRequireDefault(require("../package.json"));
 var _FSCache = require("./FSCache");
+function _logger() {
+  const data = require("@atlaspack/logger");
+  _logger = function () {
+    return data;
+  };
+  return data;
+}
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 // $FlowFixMe
+const ncpAsync = (0, _util().promisify)(_ncp().default);
 class LmdbWrapper {
   constructor(lmdb) {
     this.lmdb = lmdb;
-
-    // $FlowFixMe
-    this[Symbol.dispose] = () => {
-      this.lmdb.close();
-    };
   }
   has(key) {
     return this.lmdb.hasSync(key);
@@ -80,7 +97,19 @@ class LmdbWrapper {
     const buffer = typeof value === 'string' ? Buffer.from(value) : value;
     await this.lmdb.put(key, buffer);
   }
-  resetReadTxn() {}
+  *keys() {
+    const PAGE_SIZE = 10000000;
+    let currentKeys = this.lmdb.keysSync(0, PAGE_SIZE);
+    while (currentKeys.length > 0) {
+      for (const key of currentKeys) {
+        yield key;
+      }
+      currentKeys = this.lmdb.keysSync(currentKeys.length, PAGE_SIZE);
+    }
+  }
+  compact(targetPath) {
+    this.lmdb.compact(targetPath);
+  }
 }
 exports.LmdbWrapper = LmdbWrapper;
 function open(directory
@@ -94,9 +123,14 @@ function open(directory
 }
 const pipeline = (0, _util().promisify)(_stream().default.pipeline);
 class LMDBLiteCache {
+  /**
+   * Directory where we store raw files.
+   */
+
   constructor(cacheDir) {
-    this.fs = new (_fs().NodeFS)();
+    this.fs = new (_fs2().NodeFS)();
     this.dir = cacheDir;
+    this.cacheFilesDirectory = _path().default.join(cacheDir, 'files');
     this.fsCache = new _FSCache.FSCache(this.fs, cacheDir);
     this.store = open(cacheDir, {
       name: 'parcel-cache',
@@ -115,6 +149,7 @@ class LMDBLiteCache {
     if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
       await this.fsCache.ensure();
     }
+    await this.fs.mkdirp(this.cacheFilesDirectory);
     return Promise.resolve();
   }
   serialize() {
@@ -139,10 +174,20 @@ class LMDBLiteCache {
     await this.setBlob(key, (0, _buildCache().serialize)(value));
   }
   getStream(key) {
-    return this.fs.createReadStream(_path().default.join(this.dir, key));
+    if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
+      return this.fs.createReadStream(_path().default.join(this.dir, key));
+    }
+    return _fs().default.createReadStream(this.getFileKey(key));
   }
-  setStream(key, stream) {
-    return pipeline(stream, this.fs.createWriteStream(_path().default.join(this.dir, key)));
+  async setStream(key, stream) {
+    if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
+      return pipeline(stream, this.fs.createWriteStream(_path().default.join(this.dir, key)));
+    }
+    const filePath = this.getFileKey(key);
+    await _fs().default.promises.mkdir(_path().default.dirname(filePath), {
+      recursive: true
+    });
+    return pipeline(stream, _fs().default.createWriteStream(filePath));
   }
 
   // eslint-disable-next-line require-await
@@ -162,34 +207,27 @@ class LMDBLiteCache {
   getBuffer(key) {
     return Promise.resolve(this.store.get(key));
   }
-  #getFilePath(key, index) {
-    return _path().default.join(this.dir, `${key}-${index}`);
-  }
   hasLargeBlob(key) {
     if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
       return this.fsCache.hasLargeBlob(key);
     }
-    return this.has(key);
+    return _fs().default.promises.access(this.getFileKey(key), _fs().default.constants.F_OK).then(() => true).catch(() => false);
   }
-
-  /**
-   * @deprecated Use getBlob instead.
-   */
   getLargeBlob(key) {
     if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
       return this.fsCache.getLargeBlob(key);
     }
-    return Promise.resolve(this.getBlobSync(key));
+    return _fs().default.promises.readFile(this.getFileKey(key));
   }
-
-  /**
-   * @deprecated Use setBlob instead.
-   */
-  setLargeBlob(key, contents, options) {
+  async setLargeBlob(key, contents, options) {
     if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
       return this.fsCache.setLargeBlob(key, contents, options);
     }
-    return this.setBlob(key, contents);
+    const targetPath = this.getFileKey(key);
+    await _fs().default.promises.mkdir(_path().default.dirname(targetPath), {
+      recursive: true
+    });
+    return _fs().default.promises.writeFile(targetPath, contents);
   }
 
   /**
@@ -201,12 +239,55 @@ class LMDBLiteCache {
     }
     return this.store.delete(key);
   }
-  refresh() {
-    // Reset the read transaction for the store. This guarantees that
-    // the next read will see the latest changes to the store.
-    // Useful in scenarios where reads and writes are multi-threaded.
-    // See https://github.com/kriszyp/lmdb-js#resetreadtxn-void
-    this.store.resetReadTxn();
+  keys() {
+    return this.store.keys();
+  }
+  async compact(targetPath) {
+    await _fs().default.promises.mkdir(targetPath, {
+      recursive: true
+    });
+    const files = await _fs().default.promises.readdir(this.dir);
+    // copy all files except data.mdb and lock.mdb to the target path (recursive)
+    for (const file of files) {
+      const filePath = _path().default.join(this.dir, file);
+      if (file === 'data.mdb' || file === 'lock.mdb') {
+        continue;
+      }
+      await ncpAsync(filePath, _path().default.join(targetPath, file));
+    }
+    this.store.compact(_path().default.join(targetPath, 'data.mdb'));
+  }
+  refresh() {}
+
+  /**
+   * Streams, packages are stored in files instead of LMDB.
+   *
+   * On this case, if a cache key happens to have a parent traversal, ../..
+   * it is treated specially
+   *
+   * That is, something/../something and something are meant to be different
+   * keys.
+   *
+   * Plus we do not want to store values outside of the cache directory.
+   */
+  getFileKey(key) {
+    const cleanKey = key.split('/').map(part => {
+      if (part === '..') {
+        return '$$__parent_dir$$';
+      }
+      return part;
+    }).join('/');
+    return _path().default.join(this.cacheFilesDirectory, cleanKey);
+  }
+  async clear() {
+    await (0, _logger().instrumentAsync)('LMDBLiteCache::clear', async () => {
+      const keys = await this.keys();
+      for (const key of keys) {
+        await this.store.delete(key);
+      }
+      await this.fs.rimraf(this.cacheFilesDirectory);
+      await this.fs.mkdirp(this.cacheFilesDirectory);
+    });
   }
 }
 exports.LMDBLiteCache = LMDBLiteCache;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@atlaspack/cache",
   "description": "Interface for defining caches and file-system, IDB and LMDB implementations.",
-  "version": "3.1.1-canary.0+56fd82cd5",
+  "version": "3.1.1-canary.100+f609bf49f",
   "license": "(MIT OR Apache-2.0)",
   "type": "commonjs",
   "publishConfig": {
@@ -23,12 +23,13 @@
     "check-ts": "tsc --noEmit index.d.ts"
   },
   "dependencies": {
-    "@atlaspack/build-cache": "2.13.3-canary.68+56fd82cd5",
-    "@atlaspack/feature-flags": "2.14.1-canary.68+56fd82cd5",
-    "@atlaspack/fs": "2.14.5-canary.0+56fd82cd5",
-    "@atlaspack/logger": "2.14.5-canary.0+56fd82cd5",
-    "@atlaspack/rust": "3.2.1-canary.0+56fd82cd5",
-    "@atlaspack/utils": "2.14.5-canary.0+56fd82cd5"
+    "@atlaspack/build-cache": "2.13.3-canary.168+f609bf49f",
+    "@atlaspack/feature-flags": "2.14.1-canary.168+f609bf49f",
+    "@atlaspack/fs": "2.14.5-canary.100+f609bf49f",
+    "@atlaspack/logger": "2.14.5-canary.100+f609bf49f",
+    "@atlaspack/rust": "3.2.1-canary.100+f609bf49f",
+    "@atlaspack/utils": "2.14.5-canary.100+f609bf49f",
+    "ncp": "^2.0.0"
   },
   "devDependencies": {
     "idb": "^5.0.8"
@@ -36,5 +37,5 @@
   "browser": {
     "./src/IDBCache.js": "./src/IDBCache.browser.js"
   },
-  "gitHead": "56fd82cd544c4c9096be6ebea8261e714435de09"
+  "gitHead": "f609bf49ffa3984c0ff81d4853a5c850aaee5fce"
 }

package/src/FSCache.js

@@ -9,6 +9,8 @@ import stream from 'stream';
 import path from 'path';
 import {promisify} from 'util';
 
+import {hashString} from '@atlaspack/rust';
+import {getFeatureFlag} from '@atlaspack/feature-flags';
 import logger from '@atlaspack/logger';
 import {
   deserialize,
@@ -51,6 +53,10 @@ export class FSCache implements Cache {
   }
 
   _getCachePath(cacheId: string): FilePath {
+    if (getFeatureFlag('cachePerformanceImprovements')) {
+      const cleanId = hashString(cacheId);
+      return path.join(this.dir, cleanId.slice(0, 2), cleanId.slice(2));
+    }
     return path.join(this.dir, cacheId.slice(0, 2), cacheId.slice(2));
   }
 
@@ -90,6 +96,9 @@ export class FSCache implements Cache {
   }
 
   #getFilePath(key: string, index: number): string {
+    if (getFeatureFlag('cachePerformanceImprovements')) {
+      return path.join(this.dir, `${hashString(key)}-${index}`);
+    }
     return path.join(this.dir, `${key}-${index}`);
   }
 

package/src/LMDBLiteCache.js

@@ -10,17 +10,18 @@ import {Lmdb} from '@atlaspack/rust';
 import type {FilePath} from '@atlaspack/types';
 import type {Cache} from './types';
 import type {Readable, Writable} from 'stream';
-
+import fs from 'fs';
+import ncp from 'ncp';
+import {promisify} from 'util';
 import stream from 'stream';
 import path from 'path';
-import {promisify} from 'util';
-
 import {NodeFS} from '@atlaspack/fs';
-
 // $FlowFixMe
 import packageJson from '../package.json';
-
 import {FSCache} from './FSCache';
+import {instrumentAsync} from '@atlaspack/logger';
+
+const ncpAsync = promisify(ncp);
 
 interface DBOpenOptions {
   name: string;
@@ -35,11 +36,6 @@ export class LmdbWrapper {
 
   constructor(lmdb: Lmdb) {
     this.lmdb = lmdb;
-
-    // $FlowFixMe
-    this[Symbol.dispose] = () => {
-      this.lmdb.close();
-    };
   }
 
   has(key: string): boolean {
@@ -60,7 +56,21 @@ export class LmdbWrapper {
     await this.lmdb.put(key, buffer);
   }
 
-  resetReadTxn() {}
+  *keys(): Iterable<string> {
+    const PAGE_SIZE = 10000000;
+
+    let currentKeys = this.lmdb.keysSync(0, PAGE_SIZE);
+    while (currentKeys.length > 0) {
+      for (const key of currentKeys) {
+        yield key;
+      }
+      currentKeys = this.lmdb.keysSync(currentKeys.length, PAGE_SIZE);
+    }
+  }
+
+  compact(targetPath: string) {
+    this.lmdb.compact(targetPath);
+  }
 }
 
 export function open(
@@ -93,10 +103,15 @@ export class LMDBLiteCache implements Cache {
   dir: FilePath;
   store: LmdbWrapper;
   fsCache: FSCache;
+  /**
+   * Directory where we store raw files.
+   */
+  cacheFilesDirectory: FilePath;
 
   constructor(cacheDir: FilePath) {
     this.fs = new NodeFS();
     this.dir = cacheDir;
+    this.cacheFilesDirectory = path.join(cacheDir, 'files');
     this.fsCache = new FSCache(this.fs, cacheDir);
 
     this.store = open(cacheDir, {
@@ -117,6 +132,7 @@ export class LMDBLiteCache implements Cache {
     if (!getFeatureFlag('cachePerformanceImprovements')) {
       await this.fsCache.ensure();
     }
+    await this.fs.mkdirp(this.cacheFilesDirectory);
     return Promise.resolve();
   }
 
@@ -148,14 +164,24 @@ export class LMDBLiteCache implements Cache {
   }
 
   getStream(key: string): Readable {
-    return this.fs.createReadStream(path.join(this.dir, key));
+    if (!getFeatureFlag('cachePerformanceImprovements')) {
+      return this.fs.createReadStream(path.join(this.dir, key));
+    }
+
+    return fs.createReadStream(this.getFileKey(key));
   }
 
-  setStream(key: string, stream: Readable): Promise<void> {
-    return pipeline(
-      stream,
-      this.fs.createWriteStream(path.join(this.dir, key)),
-    );
+  async setStream(key: string, stream: Readable): Promise<void> {
+    if (!getFeatureFlag('cachePerformanceImprovements')) {
+      return pipeline(
+        stream,
+        this.fs.createWriteStream(path.join(this.dir, key)),
+      );
+    }
+
+    const filePath = this.getFileKey(key);
+    await fs.promises.mkdir(path.dirname(filePath), {recursive: true});
+    return pipeline(stream, fs.createWriteStream(filePath));
   }
 
   // eslint-disable-next-line require-await
@@ -179,31 +205,25 @@ export class LMDBLiteCache implements Cache {
     return Promise.resolve(this.store.get(key));
   }
 
-  #getFilePath(key: string, index: number): string {
-    return path.join(this.dir, `${key}-${index}`);
-  }
-
   hasLargeBlob(key: string): Promise<boolean> {
     if (!getFeatureFlag('cachePerformanceImprovements')) {
       return this.fsCache.hasLargeBlob(key);
     }
-    return this.has(key);
+
+    return fs.promises
+      .access(this.getFileKey(key), fs.constants.F_OK)
+      .then(() => true)
+      .catch(() => false);
   }
 
-  /**
-   * @deprecated Use getBlob instead.
-   */
   getLargeBlob(key: string): Promise<Buffer> {
     if (!getFeatureFlag('cachePerformanceImprovements')) {
       return this.fsCache.getLargeBlob(key);
     }
-    return Promise.resolve(this.getBlobSync(key));
+    return fs.promises.readFile(this.getFileKey(key));
   }
 
-  /**
-   * @deprecated Use setBlob instead.
-   */
-  setLargeBlob(
+  async setLargeBlob(
     key: string,
     contents: Buffer | string,
     options?: {|signal?: AbortSignal|},
@@ -211,7 +231,10 @@ export class LMDBLiteCache implements Cache {
     if (!getFeatureFlag('cachePerformanceImprovements')) {
       return this.fsCache.setLargeBlob(key, contents, options);
     }
-    return this.setBlob(key, contents);
+
+    const targetPath = this.getFileKey(key);
+    await fs.promises.mkdir(path.dirname(targetPath), {recursive: true});
+    return fs.promises.writeFile(targetPath, contents);
   }
 
   /**
@@ -225,12 +248,64 @@ export class LMDBLiteCache implements Cache {
     return this.store.delete(key);
   }
 
-  refresh(): void {
-    // Reset the read transaction for the store. This guarantees that
-    // the next read will see the latest changes to the store.
-    // Useful in scenarios where reads and writes are multi-threaded.
-    // See https://github.com/kriszyp/lmdb-js#resetreadtxn-void
-    this.store.resetReadTxn();
+  keys(): Iterable<string> {
+    return this.store.keys();
+  }
+
+  async compact(targetPath: string): Promise<void> {
+    await fs.promises.mkdir(targetPath, {recursive: true});
+
+    const files = await fs.promises.readdir(this.dir);
+    // copy all files except data.mdb and lock.mdb to the target path (recursive)
+    for (const file of files) {
+      const filePath = path.join(this.dir, file);
+
+      if (file === 'data.mdb' || file === 'lock.mdb') {
+        continue;
+      }
+
+      await ncpAsync(filePath, path.join(targetPath, file));
+    }
+
+    this.store.compact(path.join(targetPath, 'data.mdb'));
+  }
+
+  refresh(): void {}
+
+  /**
+   * Streams, packages are stored in files instead of LMDB.
+   *
+   * On this case, if a cache key happens to have a parent traversal, ../..
+   * it is treated specially
+   *
+   * That is, something/../something and something are meant to be different
+   * keys.
+   *
+   * Plus we do not want to store values outside of the cache directory.
+   */
+  getFileKey(key: string): string {
+    const cleanKey = key
+      .split('/')
+      .map((part) => {
+        if (part === '..') {
+          return '$$__parent_dir$$';
+        }
+        return part;
+      })
+      .join('/');
+    return path.join(this.cacheFilesDirectory, cleanKey);
+  }
+
+  async clear(): Promise<void> {
+    await instrumentAsync('LMDBLiteCache::clear', async () => {
+      const keys = await this.keys();
+      for (const key of keys) {
+        await this.store.delete(key);
+      }
+
+      await this.fs.rimraf(this.cacheFilesDirectory);
+      await this.fs.mkdirp(this.cacheFilesDirectory);
+    });
   }
 }
 

package/test/LMDBLiteCache.test.js

@@ -1,20 +1,30 @@
 // @flow
+
+import * as fs from 'fs';
 import * as path from 'path';
 import {tmpdir} from 'os';
 import {LMDBLiteCache} from '../src/index';
 import {deserialize, serialize} from 'v8';
 import assert from 'assert';
+import {Worker} from 'worker_threads';
+import {initializeMonitoring} from '@atlaspack/rust';
 
 const cacheDir = path.join(tmpdir(), 'lmdb-lite-cache-tests');
 
 describe('LMDBLiteCache', () => {
+  let cache;
+
+  beforeEach(async () => {
+    await fs.promises.rm(cacheDir, {recursive: true, force: true});
+  });
+
   it('can be constructed', async () => {
-    const cache = new LMDBLiteCache(cacheDir);
+    cache = new LMDBLiteCache(cacheDir);
     await cache.ensure();
   });
 
   it('can retrieve keys', async () => {
-    const cache = new LMDBLiteCache(cacheDir);
+    cache = new LMDBLiteCache(cacheDir);
     await cache.ensure();
     await cache.setBlob('key', Buffer.from(serialize({value: 42})));
     const buffer = await cache.getBlob('key');
@@ -23,11 +33,211 @@ describe('LMDBLiteCache', () => {
   });
 
   it('can retrieve keys synchronously', async () => {
-    const cache = new LMDBLiteCache(cacheDir);
+    cache = new LMDBLiteCache(path.join(cacheDir, 'retrieve_keys_test'));
     await cache.ensure();
-    cache.setBlob('key', Buffer.from(serialize({value: 42})));
+    await cache.setBlob('key', Buffer.from(serialize({value: 42})));
     const buffer = cache.getBlobSync('key');
     const result = deserialize(buffer);
     assert.equal(result.value, 42);
   });
+
+  it('can iterate over keys', async () => {
+    cache = new LMDBLiteCache(path.join(cacheDir, 'keys_test'));
+    await cache.ensure();
+    await cache.setBlob('key1', Buffer.from(serialize({value: 42})));
+    await cache.setBlob('key2', Buffer.from(serialize({value: 43})));
+    const keys = cache.keys();
+    assert.deepEqual(Array.from(keys), ['key1', 'key2']);
+  });
+
+  it('can compact databases', async () => {
+    cache = new LMDBLiteCache(path.join(cacheDir, 'compact_test'));
+    await cache.ensure();
+    await cache.setBlob('key1', Buffer.from(serialize({value: 42})));
+    await cache.setBlob('key2', Buffer.from(serialize({value: 43})));
+    await cache.compact(path.join(cacheDir, 'compact_test_compacted'));
+
+    cache = new LMDBLiteCache(path.join(cacheDir, 'compact_test_compacted'));
+    await cache.ensure();
+    const keys = cache.keys();
+    assert.deepEqual(Array.from(keys), ['key1', 'key2']);
+  });
+
+  describe('getFileKey', () => {
+    it('should return the correct key', () => {
+      const target = path.join(cacheDir, 'test-file-keys');
+      const cache = new LMDBLiteCache(target);
+      const key = cache.getFileKey('key');
+      assert.equal(key, path.join(target, 'files', 'key'));
+    });
+
+    it('should return the correct key for a key with a parent traversal', () => {
+      const target = path.join(cacheDir, 'test-parent-keys');
+      cache = new LMDBLiteCache(target);
+      const key = cache.getFileKey('../../key');
+      assert.equal(
+        key,
+        path.join(target, 'files', '$$__parent_dir$$/$$__parent_dir$$/key'),
+      );
+    });
+  });
+
+  it('can be closed and re-opened', async () => {
+    cache = new LMDBLiteCache(path.join(cacheDir, 'close_and_reopen_test'));
+    await cache.ensure();
+    await cache.setBlob('key', Buffer.from(serialize({value: 42})));
+    cache = new LMDBLiteCache(path.join(cacheDir, 'close_and_reopen_test'));
+    await cache.ensure();
+    const buffer = await cache.getBlob('key');
+    const result = deserialize(buffer);
+    assert.equal(result.value, 42);
+  });
+
+  it('should NOT fail when trying to open the same database twice', async () => {
+    const testDir = path.join(cacheDir, 'double_open_test');
+    const cache1 = new LMDBLiteCache(testDir);
+    await cache1.ensure();
+
+    assert.doesNotThrow(() => {
+      new LMDBLiteCache(testDir);
+    });
+  });
+
+  it('should NOT fail when trying to open after GC', async () => {
+    const testDir = path.join(cacheDir, 'gc_test');
+
+    let cache1 = new LMDBLiteCache(testDir);
+    await cache1.ensure();
+    await cache1.setBlob('key', Buffer.from(serialize({value: 42})));
+
+    cache1 = null;
+
+    if (global.gc) {
+      global.gc();
+    }
+
+    assert.doesNotThrow(() => {
+      new LMDBLiteCache(testDir);
+    });
+  });
+
+  it('should handle rapid open/close cycles', async () => {
+    const testDir = path.join(cacheDir, 'rapid_cycles_test');
+
+    for (let i = 0; i < 10; i++) {
+      const cache = new LMDBLiteCache(testDir);
+      await cache.ensure();
+      await cache.setBlob(`key${i}`, Buffer.from(serialize({value: i})));
+
+      await new Promise((resolve) => setTimeout(resolve, 10));
+    }
+
+    const finalCache = new LMDBLiteCache(testDir);
+    await finalCache.ensure();
+    const buffer = await finalCache.getBlob('key9');
+    const result = deserialize(buffer);
+    assert.equal(result.value, 9);
+  });
+
+  it('should work when there are multiple node.js worker threads accessing the same database', async function () {
+    this.timeout(40000);
+
+    try {
+      initializeMonitoring();
+    } catch (error) {
+      /* empty */
+    }
+
+    const testDir = path.join(cacheDir, 'worker_threads_test');
+
+    let cache = new LMDBLiteCache(testDir);
+    await cache.set('main_thread_key', {
+      mainThreadId: 0,
+      hello: 'world',
+    });
+    setTimeout(() => {
+      cache = null;
+
+      if (global.gc) {
+        global.gc();
+      }
+    }, Math.random() * 300);
+
+    const numWorkers = 10;
+
+    const workers = [];
+    const responsePromises = [];
+    for (let i = 0; i < numWorkers; i++) {
+      const worker = new Worker(path.join(__dirname, 'workerThreadsTest.js'), {
+        workerData: {
+          cacheDir: testDir,
+        },
+      });
+      workers.push(worker);
+
+      const responsePromise = new Promise((resolve, reject) => {
+        worker.addListener('error', (error: Error) => {
+          reject(error);
+        });
+        worker.addListener('message', (message) => {
+          resolve(message);
+        });
+      });
+
+      worker.addListener('message', (message) => {
+        // eslint-disable-next-line no-console
+        console.log('Worker message', message);
+      });
+      worker.addListener('online', () => {
+        worker.postMessage({
+          type: 'go',
+        });
+      });
+
+      responsePromises.push(responsePromise);
+    }
+
+    // eslint-disable-next-line no-console
+    console.log('Waiting for responses');
+    const responses = await Promise.all(responsePromises);
+
+    // eslint-disable-next-line no-console
+    console.log('Responses received');
+    for (const [index, response] of responses.entries()) {
+      const worker = workers[index];
+
+      assert.deepEqual(
+        response,
+        {
+          mainThreadData: {
+            mainThreadId: 0,
+            hello: 'world',
+          },
+          workerId: worker.threadId,
+        },
+        `worker_${index} - Worker ${worker.threadId} should have received the correct data`,
+      );
+    }
+
+    // eslint-disable-next-line no-console
+    console.log('Getting main thread key');
+    cache = new LMDBLiteCache(testDir);
+    const data = await cache?.get('main_thread_key');
+    assert.deepEqual(data, {
+      mainThreadId: 0,
+      hello: 'world',
+    });
+
+    // eslint-disable-next-line no-console
+    console.log('Getting worker keys');
+    for (const worker of workers) {
+      const data = await cache?.get(`worker_key/${worker.threadId}`);
+      assert.deepEqual(data, {
+        workerId: worker.threadId,
+      });
+
+      await new Promise((resolve) => setTimeout(resolve, 500));
+      worker.terminate();
+    }
+  });
 });

package/test/workerThreadsTest.js

@@ -0,0 +1,42 @@
+/* eslint-disable no-inner-declarations */
+
+require('@atlaspack/babel-register');
+const {
+  workerData,
+  threadId,
+  parentPort,
+  isMainThread,
+} = require('worker_threads');
+const {LMDBLiteCache} = require('../src/index');
+
+if (!isMainThread) {
+  const cache = new LMDBLiteCache(workerData.cacheDir);
+
+  async function onMessage() {
+    try {
+      cache.set(`worker_key/${threadId}`, {
+        workerId: threadId,
+      });
+
+      const data = await cache.get('main_thread_key');
+
+      parentPort.postMessage({
+        mainThreadData: data,
+        workerId: threadId,
+      });
+
+      setTimeout(() => {
+        parentPort.postMessage({
+          type: 'close',
+          workerId: threadId,
+        });
+      }, Math.random() * 200);
+    } catch (error) {
+      parentPort.postMessage({
+        error: error.message,
+      });
+    }
+  }
+
+  parentPort.on('message', onMessage);
+}