@atlaspack/cache 3.1.1-canary.41 → 3.1.1-canary.43

package/lib/FSCache.js

@@ -25,6 +25,20 @@ function _util() {
   };
   return data;
 }
+function _rust() {
+  const data = require("@atlaspack/rust");
+  _rust = function () {
+    return data;
+  };
+  return data;
+}
+function _featureFlags() {
+  const data = require("@atlaspack/feature-flags");
+  _featureFlags = function () {
+    return data;
+  };
+  return data;
+}
 function _logger() {
   const data = _interopRequireDefault(require("@atlaspack/logger"));
   _logger = function () {
@@ -62,6 +76,10 @@ class FSCache {
     await Promise.all(dirPromises);
   }
   _getCachePath(cacheId) {
+    if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
+      const cleanId = (0, _rust().hashString)(cacheId);
+      return _path().default.join(this.dir, cleanId.slice(0, 2), cleanId.slice(2));
+    }
     return _path().default.join(this.dir, cacheId.slice(0, 2), cacheId.slice(2));
   }
   getStream(key) {
@@ -91,6 +109,9 @@ class FSCache {
     }
   }
   #getFilePath(key, index) {
+    if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
+      return _path().default.join(this.dir, `${(0, _rust().hashString)(key)}-${index}`);
+    }
     return _path().default.join(this.dir, `${key}-${index}`);
   }
   async #unlinkChunks(key, index) {

package/lib/LMDBLiteCache.js

@@ -70,6 +70,13 @@ function _fs2() {
 }
 var _package = _interopRequireDefault(require("../package.json"));
 var _FSCache = require("./FSCache");
+function _logger() {
+  const data = require("@atlaspack/logger");
+  _logger = function () {
+    return data;
+  };
+  return data;
+}
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 // $FlowFixMe
 const ncpAsync = (0, _util().promisify)(_ncp().default);
@@ -121,9 +128,14 @@ function open(directory
 }
 const pipeline = (0, _util().promisify)(_stream().default.pipeline);
 class LMDBLiteCache {
+  /**
+   * Directory where we store raw files.
+   */
+
   constructor(cacheDir) {
     this.fs = new (_fs2().NodeFS)();
     this.dir = cacheDir;
+    this.cacheFilesDirectory = _path().default.join(cacheDir, 'files');
     this.fsCache = new _FSCache.FSCache(this.fs, cacheDir);
     this.store = open(cacheDir, {
       name: 'parcel-cache',
@@ -142,6 +154,7 @@ class LMDBLiteCache {
     if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
       await this.fsCache.ensure();
     }
+    await this.fs.mkdirp(this.cacheFilesDirectory);
     return Promise.resolve();
   }
   serialize() {
@@ -166,10 +179,20 @@ class LMDBLiteCache {
     await this.setBlob(key, (0, _buildCache().serialize)(value));
   }
   getStream(key) {
-    return this.fs.createReadStream(_path().default.join(this.dir, key));
+    if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
+      return this.fs.createReadStream(_path().default.join(this.dir, key));
+    }
+    return _fs().default.createReadStream(this.getFileKey(key));
   }
-  setStream(key, stream) {
-    return pipeline(stream, this.fs.createWriteStream(_path().default.join(this.dir, key)));
+  async setStream(key, stream) {
+    if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
+      return pipeline(stream, this.fs.createWriteStream(_path().default.join(this.dir, key)));
+    }
+    const filePath = this.getFileKey(key);
+    await _fs().default.promises.mkdir(_path().default.dirname(filePath), {
+      recursive: true
+    });
+    return pipeline(stream, _fs().default.createWriteStream(filePath));
   }
 
   // eslint-disable-next-line require-await
@@ -189,34 +212,27 @@ class LMDBLiteCache {
   getBuffer(key) {
     return Promise.resolve(this.store.get(key));
   }
-  #getFilePath(key, index) {
-    return _path().default.join(this.dir, `${key}-${index}`);
-  }
   hasLargeBlob(key) {
     if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
       return this.fsCache.hasLargeBlob(key);
     }
-    return this.has(key);
+    return _fs().default.promises.access(this.getFileKey(key), _fs().default.constants.F_OK).then(() => true).catch(() => false);
   }
-
-  /**
-   * @deprecated Use getBlob instead.
-   */
   getLargeBlob(key) {
     if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
       return this.fsCache.getLargeBlob(key);
     }
-    return Promise.resolve(this.getBlobSync(key));
+    return _fs().default.promises.readFile(this.getFileKey(key));
   }
-
-  /**
-   * @deprecated Use setBlob instead.
-   */
-  setLargeBlob(key, contents, options) {
+  async setLargeBlob(key, contents, options) {
     if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
       return this.fsCache.setLargeBlob(key, contents, options);
     }
-    return this.setBlob(key, contents);
+    const targetPath = this.getFileKey(key);
+    await _fs().default.promises.mkdir(_path().default.dirname(targetPath), {
+      recursive: true
+    });
+    return _fs().default.promises.writeFile(targetPath, contents);
   }
 
   /**
@@ -247,6 +263,37 @@ class LMDBLiteCache {
     this.store.compact(_path().default.join(targetPath, 'data.mdb'));
   }
   refresh() {}
+
+  /**
+   * Streams, packages are stored in files instead of LMDB.
+   *
+   * On this case, if a cache key happens to have a parent traversal, ../..
+   * it is treated specially
+   *
+   * That is, something/../something and something are meant to be different
+   * keys.
+   *
+   * Plus we do not want to store values outside of the cache directory.
+   */
+  getFileKey(key) {
+    const cleanKey = key.split('/').map(part => {
+      if (part === '..') {
+        return '$$__parent_dir$$';
+      }
+      return part;
+    }).join('/');
+    return _path().default.join(this.cacheFilesDirectory, cleanKey);
+  }
+  async clear() {
+    await (0, _logger().instrumentAsync)('LMDBLiteCache::clear', async () => {
+      const keys = await this.keys();
+      for (const key of keys) {
+        await this.store.delete(key);
+      }
+      await this.fs.rimraf(this.cacheFilesDirectory);
+      await this.fs.mkdirp(this.cacheFilesDirectory);
+    });
+  }
 }
 exports.LMDBLiteCache = LMDBLiteCache;
 (0, _buildCache().registerSerializableClass)(`${_package.default.version}:LMDBLiteCache`, LMDBLiteCache);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@atlaspack/cache",
   "description": "Interface for defining caches and file-system, IDB and LMDB implementations.",
-  "version": "3.1.1-canary.41+212de35c7",
+  "version": "3.1.1-canary.43+0e0458eed",
   "license": "(MIT OR Apache-2.0)",
   "type": "commonjs",
   "publishConfig": {
@@ -23,12 +23,12 @@
     "check-ts": "tsc --noEmit index.d.ts"
   },
   "dependencies": {
-    "@atlaspack/build-cache": "2.13.3-canary.109+212de35c7",
-    "@atlaspack/feature-flags": "2.14.1-canary.109+212de35c7",
-    "@atlaspack/fs": "2.14.5-canary.41+212de35c7",
-    "@atlaspack/logger": "2.14.5-canary.41+212de35c7",
-    "@atlaspack/rust": "3.2.1-canary.41+212de35c7",
-    "@atlaspack/utils": "2.14.5-canary.41+212de35c7",
+    "@atlaspack/build-cache": "2.13.3-canary.111+0e0458eed",
+    "@atlaspack/feature-flags": "2.14.1-canary.111+0e0458eed",
+    "@atlaspack/fs": "2.14.5-canary.43+0e0458eed",
+    "@atlaspack/logger": "2.14.5-canary.43+0e0458eed",
+    "@atlaspack/rust": "3.2.1-canary.43+0e0458eed",
+    "@atlaspack/utils": "2.14.5-canary.43+0e0458eed",
     "ncp": "^2.0.0"
   },
   "devDependencies": {
@@ -37,5 +37,5 @@
   "browser": {
     "./src/IDBCache.js": "./src/IDBCache.browser.js"
   },
-  "gitHead": "212de35c70ca46b0510c94fbda916892b667692c"
+  "gitHead": "0e0458eedab6e398b375b073439f27f8bcd0b490"
 }

package/src/FSCache.js

@@ -9,6 +9,8 @@ import stream from 'stream';
 import path from 'path';
 import {promisify} from 'util';
 
+import {hashString} from '@atlaspack/rust';
+import {getFeatureFlag} from '@atlaspack/feature-flags';
 import logger from '@atlaspack/logger';
 import {
   deserialize,
@@ -51,6 +53,10 @@ export class FSCache implements Cache {
   }
 
   _getCachePath(cacheId: string): FilePath {
+    if (getFeatureFlag('cachePerformanceImprovements')) {
+      const cleanId = hashString(cacheId);
+      return path.join(this.dir, cleanId.slice(0, 2), cleanId.slice(2));
+    }
     return path.join(this.dir, cacheId.slice(0, 2), cacheId.slice(2));
   }
 
@@ -90,6 +96,9 @@ export class FSCache implements Cache {
   }
 
   #getFilePath(key: string, index: number): string {
+    if (getFeatureFlag('cachePerformanceImprovements')) {
+      return path.join(this.dir, `${hashString(key)}-${index}`);
+    }
     return path.join(this.dir, `${key}-${index}`);
   }
 

package/src/LMDBLiteCache.js

@@ -19,6 +19,7 @@ import {NodeFS} from '@atlaspack/fs';
 // $FlowFixMe
 import packageJson from '../package.json';
 import {FSCache} from './FSCache';
+import {instrumentAsync} from '@atlaspack/logger';
 
 const ncpAsync = promisify(ncp);
 
@@ -107,10 +108,15 @@ export class LMDBLiteCache implements Cache {
   dir: FilePath;
   store: LmdbWrapper;
   fsCache: FSCache;
+  /**
+   * Directory where we store raw files.
+   */
+  cacheFilesDirectory: FilePath;
 
   constructor(cacheDir: FilePath) {
     this.fs = new NodeFS();
     this.dir = cacheDir;
+    this.cacheFilesDirectory = path.join(cacheDir, 'files');
     this.fsCache = new FSCache(this.fs, cacheDir);
 
     this.store = open(cacheDir, {
@@ -131,6 +137,7 @@ export class LMDBLiteCache implements Cache {
     if (!getFeatureFlag('cachePerformanceImprovements')) {
       await this.fsCache.ensure();
     }
+    await this.fs.mkdirp(this.cacheFilesDirectory);
     return Promise.resolve();
   }
 
@@ -162,14 +169,24 @@ export class LMDBLiteCache implements Cache {
   }
 
   getStream(key: string): Readable {
-    return this.fs.createReadStream(path.join(this.dir, key));
+    if (!getFeatureFlag('cachePerformanceImprovements')) {
+      return this.fs.createReadStream(path.join(this.dir, key));
+    }
+
+    return fs.createReadStream(this.getFileKey(key));
   }
 
-  setStream(key: string, stream: Readable): Promise<void> {
-    return pipeline(
-      stream,
-      this.fs.createWriteStream(path.join(this.dir, key)),
-    );
+  async setStream(key: string, stream: Readable): Promise<void> {
+    if (!getFeatureFlag('cachePerformanceImprovements')) {
+      return pipeline(
+        stream,
+        this.fs.createWriteStream(path.join(this.dir, key)),
+      );
+    }
+
+    const filePath = this.getFileKey(key);
+    await fs.promises.mkdir(path.dirname(filePath), {recursive: true});
+    return pipeline(stream, fs.createWriteStream(filePath));
   }
 
   // eslint-disable-next-line require-await
@@ -193,31 +210,25 @@ export class LMDBLiteCache implements Cache {
     return Promise.resolve(this.store.get(key));
   }
 
-  #getFilePath(key: string, index: number): string {
-    return path.join(this.dir, `${key}-${index}`);
-  }
-
   hasLargeBlob(key: string): Promise<boolean> {
     if (!getFeatureFlag('cachePerformanceImprovements')) {
       return this.fsCache.hasLargeBlob(key);
     }
-    return this.has(key);
+
+    return fs.promises
+      .access(this.getFileKey(key), fs.constants.F_OK)
+      .then(() => true)
+      .catch(() => false);
   }
 
-  /**
-   * @deprecated Use getBlob instead.
-   */
   getLargeBlob(key: string): Promise<Buffer> {
     if (!getFeatureFlag('cachePerformanceImprovements')) {
       return this.fsCache.getLargeBlob(key);
     }
-    return Promise.resolve(this.getBlobSync(key));
+    return fs.promises.readFile(this.getFileKey(key));
   }
 
-  /**
-   * @deprecated Use setBlob instead.
-   */
-  setLargeBlob(
+  async setLargeBlob(
     key: string,
     contents: Buffer | string,
     options?: {|signal?: AbortSignal|},
@@ -225,7 +236,10 @@ export class LMDBLiteCache implements Cache {
     if (!getFeatureFlag('cachePerformanceImprovements')) {
       return this.fsCache.setLargeBlob(key, contents, options);
     }
-    return this.setBlob(key, contents);
+
+    const targetPath = this.getFileKey(key);
+    await fs.promises.mkdir(path.dirname(targetPath), {recursive: true});
+    return fs.promises.writeFile(targetPath, contents);
   }
 
   /**
@@ -262,6 +276,42 @@ export class LMDBLiteCache implements Cache {
   }
 
   refresh(): void {}
+
+  /**
+   * Streams, packages are stored in files instead of LMDB.
+   *
+   * On this case, if a cache key happens to have a parent traversal, ../..
+   * it is treated specially
+   *
+   * That is, something/../something and something are meant to be different
+   * keys.
+   *
+   * Plus we do not want to store values outside of the cache directory.
+   */
+  getFileKey(key: string): string {
+    const cleanKey = key
+      .split('/')
+      .map((part) => {
+        if (part === '..') {
+          return '$$__parent_dir$$';
+        }
+        return part;
+      })
+      .join('/');
+    return path.join(this.cacheFilesDirectory, cleanKey);
+  }
+
+  async clear(): Promise<void> {
+    await instrumentAsync('LMDBLiteCache::clear', async () => {
+      const keys = await this.keys();
+      for (const key of keys) {
+        await this.store.delete(key);
+      }
+
+      await this.fs.rimraf(this.cacheFilesDirectory);
+      await this.fs.mkdirp(this.cacheFilesDirectory);
+    });
+  }
 }
 
 registerSerializableClass(

package/test/LMDBLiteCache.test.js

@@ -16,10 +16,6 @@ describe('LMDBLiteCache', () => {
     await fs.promises.rm(cacheDir, {recursive: true, force: true});
   });
 
-  afterEach(() => {
-    cache.getNativeRef().close();
-  });
-
   it('can be constructed', async () => {
     cache = new LMDBLiteCache(cacheDir);
     await cache.ensure();
@@ -59,11 +55,40 @@ describe('LMDBLiteCache', () => {
     await cache.setBlob('key2', Buffer.from(serialize({value: 43})));
     await cache.compact(path.join(cacheDir, 'compact_test_compacted'));
 
-    cache.getNativeRef().close();
-
     cache = new LMDBLiteCache(path.join(cacheDir, 'compact_test_compacted'));
     await cache.ensure();
     const keys = cache.keys();
     assert.deepEqual(Array.from(keys), ['key1', 'key2']);
   });
+
+  describe('getFileKey', () => {
+    it('should return the correct key', () => {
+      const target = path.join(cacheDir, 'test-file-keys');
+      const cache = new LMDBLiteCache(target);
+      const key = cache.getFileKey('key');
+      assert.equal(key, path.join(target, 'files', 'key'));
+    });
+
+    it('should return the correct key for a key with a parent traversal', () => {
+      const target = path.join(cacheDir, 'test-parent-keys');
+      cache = new LMDBLiteCache(target);
+      const key = cache.getFileKey('../../key');
+      assert.equal(
+        key,
+        path.join(target, 'files', '$$__parent_dir$$/$$__parent_dir$$/key'),
+      );
+    });
+  });
+
+  it('can be closed and re-opened', async () => {
+    cache = new LMDBLiteCache(path.join(cacheDir, 'close_and_reopen_test'));
+    await cache.ensure();
+    await cache.setBlob('key', Buffer.from(serialize({value: 42})));
+    cache.getNativeRef().close();
+    cache = new LMDBLiteCache(path.join(cacheDir, 'close_and_reopen_test'));
+    await cache.ensure();
+    const buffer = await cache.getBlob('key');
+    const result = deserialize(buffer);
+    assert.equal(result.value, 42);
+  });
 });