@atlaspack/cache 3.1.1-canary.9 → 3.1.1-dev.14

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @atlaspack/cache
 
+## 3.2.0
+
+### Minor Changes
+
+- [#531](https://github.com/atlassian-labs/atlaspack/pull/531) [`d2c50c2`](https://github.com/atlassian-labs/atlaspack/commit/d2c50c2c020888b33bb25b8690d9320c2b69e2a6) Thanks [@yamadapc](https://github.com/yamadapc)! - Add way to iterate LMDB cache keys
+
+### Patch Changes
+
+- Updated dependencies [[`11d6f16`](https://github.com/atlassian-labs/atlaspack/commit/11d6f16b6397dee2f217167e5c98b39edb63f7a7), [`e2ba0f6`](https://github.com/atlassian-labs/atlaspack/commit/e2ba0f69702656f3d1ce95ab1454e35062b13b39), [`d2c50c2`](https://github.com/atlassian-labs/atlaspack/commit/d2c50c2c020888b33bb25b8690d9320c2b69e2a6), [`46a90dc`](https://github.com/atlassian-labs/atlaspack/commit/46a90dccd019a26b222c878a92d23acc75dc67c5), [`4c17141`](https://github.com/atlassian-labs/atlaspack/commit/4c1714103dab2aa9039c488f381551d2b65d1d01)]:
+  - @atlaspack/feature-flags@2.14.3
+  - @atlaspack/rust@3.3.0
+  - @atlaspack/fs@2.15.0
+  - @atlaspack/utils@2.14.5
+  - @atlaspack/logger@2.14.5
+
 ## 3.1.0
 
 ### Minor Changes

package/lib/LMDBLiteCache.js

@@ -26,16 +26,16 @@ function _rust() {
   };
   return data;
 }
-function _stream() {
-  const data = _interopRequireDefault(require("stream"));
-  _stream = function () {
+function _fs() {
+  const data = _interopRequireDefault(require("fs"));
+  _fs = function () {
     return data;
   };
   return data;
 }
-function _path() {
-  const data = _interopRequireDefault(require("path"));
-  _path = function () {
+function _ncp() {
+  const data = _interopRequireDefault(require("ncp"));
+  _ncp = function () {
     return data;
   };
   return data;
@@ -47,9 +47,23 @@ function _util() {
   };
   return data;
 }
-function _fs() {
+function _stream() {
+  const data = _interopRequireDefault(require("stream"));
+  _stream = function () {
+    return data;
+  };
+  return data;
+}
+function _path() {
+  const data = _interopRequireDefault(require("path"));
+  _path = function () {
+    return data;
+  };
+  return data;
+}
+function _fs2() {
   const data = require("@atlaspack/fs");
-  _fs = function () {
+  _fs2 = function () {
     return data;
   };
   return data;
@@ -58,6 +72,7 @@ var _package = _interopRequireDefault(require("../package.json"));
 var _FSCache = require("./FSCache");
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 // $FlowFixMe
+const ncpAsync = (0, _util().promisify)(_ncp().default);
 class LmdbWrapper {
   constructor(lmdb) {
     this.lmdb = lmdb;
@@ -90,7 +105,9 @@ class LmdbWrapper {
       currentKeys = this.lmdb.keysSync(currentKeys.length, PAGE_SIZE);
     }
   }
-  resetReadTxn() {}
+  compact(targetPath) {
+    this.lmdb.compact(targetPath);
+  }
 }
 exports.LmdbWrapper = LmdbWrapper;
 function open(directory
@@ -105,7 +122,7 @@ function open(directory
 const pipeline = (0, _util().promisify)(_stream().default.pipeline);
 class LMDBLiteCache {
   constructor(cacheDir) {
-    this.fs = new (_fs().NodeFS)();
+    this.fs = new (_fs2().NodeFS)();
     this.dir = cacheDir;
     this.fsCache = new _FSCache.FSCache(this.fs, cacheDir);
     this.store = open(cacheDir, {
@@ -214,13 +231,22 @@ class LMDBLiteCache {
   keys() {
     return this.store.keys();
   }
-  refresh() {
-    // Reset the read transaction for the store. This guarantees that
-    // the next read will see the latest changes to the store.
-    // Useful in scenarios where reads and writes are multi-threaded.
-    // See https://github.com/kriszyp/lmdb-js#resetreadtxn-void
-    this.store.resetReadTxn();
+  async compact(targetPath) {
+    await _fs().default.promises.mkdir(targetPath, {
+      recursive: true
+    });
+    const files = await _fs().default.promises.readdir(this.dir);
+    // copy all files except data.mdb and lock.mdb to the target path (recursive)
+    for (const file of files) {
+      const filePath = _path().default.join(this.dir, file);
+      if (file === 'data.mdb' || file === 'lock.mdb') {
+        continue;
+      }
+      await ncpAsync(filePath, _path().default.join(targetPath, file));
+    }
+    this.store.compact(_path().default.join(targetPath, 'data.mdb'));
   }
+  refresh() {}
 }
 exports.LMDBLiteCache = LMDBLiteCache;
 (0, _buildCache().registerSerializableClass)(`${_package.default.version}:LMDBLiteCache`, LMDBLiteCache);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@atlaspack/cache",
   "description": "Interface for defining caches and file-system, IDB and LMDB implementations.",
-  "version": "3.1.1-canary.9+4c1714103",
+  "version": "3.1.1-dev.14+8c369e38c",
   "license": "(MIT OR Apache-2.0)",
   "type": "commonjs",
   "publishConfig": {
@@ -23,12 +23,13 @@
     "check-ts": "tsc --noEmit index.d.ts"
   },
   "dependencies": {
-    "@atlaspack/build-cache": "2.13.3-canary.77+4c1714103",
-    "@atlaspack/feature-flags": "2.14.1-canary.77+4c1714103",
-    "@atlaspack/fs": "2.14.5-canary.9+4c1714103",
-    "@atlaspack/logger": "2.14.5-canary.9+4c1714103",
-    "@atlaspack/rust": "3.2.1-canary.9+4c1714103",
-    "@atlaspack/utils": "2.14.5-canary.9+4c1714103"
+    "@atlaspack/build-cache": "2.13.3-dev.82+8c369e38c",
+    "@atlaspack/feature-flags": "2.14.1-dev.82+8c369e38c",
+    "@atlaspack/fs": "2.14.5-dev.14+8c369e38c",
+    "@atlaspack/logger": "2.14.5-dev.14+8c369e38c",
+    "@atlaspack/rust": "3.2.1-dev.14+8c369e38c",
+    "@atlaspack/utils": "2.14.5-dev.14+8c369e38c",
+    "ncp": "^2.0.0"
   },
   "devDependencies": {
     "idb": "^5.0.8"
@@ -36,5 +37,5 @@
   "browser": {
     "./src/IDBCache.js": "./src/IDBCache.browser.js"
   },
-  "gitHead": "4c1714103dab2aa9039c488f381551d2b65d1d01"
+  "gitHead": "8c369e38ccd428409811114aebd6044c27f90705"
 }

package/src/LMDBLiteCache.js

@@ -10,18 +10,18 @@ import {Lmdb} from '@atlaspack/rust';
 import type {FilePath} from '@atlaspack/types';
 import type {Cache} from './types';
 import type {Readable, Writable} from 'stream';
-
+import fs from 'fs';
+import ncp from 'ncp';
+import {promisify} from 'util';
 import stream from 'stream';
 import path from 'path';
-import {promisify} from 'util';
-
 import {NodeFS} from '@atlaspack/fs';
-
 // $FlowFixMe
 import packageJson from '../package.json';
-
 import {FSCache} from './FSCache';
 
+const ncpAsync = promisify(ncp);
+
 interface DBOpenOptions {
   name: string;
   // unused
@@ -72,7 +72,9 @@ export class LmdbWrapper {
     }
   }
 
-  resetReadTxn() {}
+  compact(targetPath: string) {
+    this.lmdb.compact(targetPath);
+  }
 }
 
 export function open(
@@ -241,13 +243,25 @@ export class LMDBLiteCache implements Cache {
     return this.store.keys();
   }
 
-  refresh(): void {
-    // Reset the read transaction for the store. This guarantees that
-    // the next read will see the latest changes to the store.
-    // Useful in scenarios where reads and writes are multi-threaded.
-    // See https://github.com/kriszyp/lmdb-js#resetreadtxn-void
-    this.store.resetReadTxn();
+  async compact(targetPath: string): Promise<void> {
+    await fs.promises.mkdir(targetPath, {recursive: true});
+
+    const files = await fs.promises.readdir(this.dir);
+    // copy all files except data.mdb and lock.mdb to the target path (recursive)
+    for (const file of files) {
+      const filePath = path.join(this.dir, file);
+
+      if (file === 'data.mdb' || file === 'lock.mdb') {
+        continue;
+      }
+
+      await ncpAsync(filePath, path.join(targetPath, file));
+    }
+
+    this.store.compact(path.join(targetPath, 'data.mdb'));
   }
+
+  refresh(): void {}
 }
 
 registerSerializableClass(

package/test/LMDBLiteCache.test.js

@@ -51,4 +51,19 @@ describe('LMDBLiteCache', () => {
     const keys = cache.keys();
     assert.deepEqual(Array.from(keys), ['key1', 'key2']);
   });
+
+  it('can compact databases', async () => {
+    cache = new LMDBLiteCache(path.join(cacheDir, 'compact_test'));
+    await cache.ensure();
+    await cache.setBlob('key1', Buffer.from(serialize({value: 42})));
+    await cache.setBlob('key2', Buffer.from(serialize({value: 43})));
+    await cache.compact(path.join(cacheDir, 'compact_test_compacted'));
+
+    cache.getNativeRef().close();
+
+    cache = new LMDBLiteCache(path.join(cacheDir, 'compact_test_compacted'));
+    await cache.ensure();
+    const keys = cache.keys();
+    assert.deepEqual(Array.from(keys), ['key1', 'key2']);
+  });
 });