@atlaspack/cache 2.12.1-dev.3398 → 2.12.1-dev.3443

package/lib/LMDBLiteCache.js

@@ -0,0 +1,174 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.LmdbWrapper = exports.LMDBLiteCache = void 0;
+exports.open = open;
+function _rust() {
+  const data = require("@atlaspack/rust");
+  _rust = function () {
+    return data;
+  };
+  return data;
+}
+function _stream() {
+  const data = _interopRequireDefault(require("stream"));
+  _stream = function () {
+    return data;
+  };
+  return data;
+}
+function _path() {
+  const data = _interopRequireDefault(require("path"));
+  _path = function () {
+    return data;
+  };
+  return data;
+}
+function _util() {
+  const data = require("util");
+  _util = function () {
+    return data;
+  };
+  return data;
+}
+function _core() {
+  const data = require("@atlaspack/core");
+  _core = function () {
+    return data;
+  };
+  return data;
+}
+function _fs() {
+  const data = require("@atlaspack/fs");
+  _fs = function () {
+    return data;
+  };
+  return data;
+}
+var _package = _interopRequireDefault(require("../package.json"));
+var _FSCache = require("./FSCache");
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+// $FlowFixMe
+class LmdbWrapper {
+  constructor(lmdb) {
+    this.lmdb = lmdb;
+
+    // $FlowFixMe
+    this[Symbol.dispose] = () => {
+      this.lmdb.close();
+    };
+  }
+  get(key) {
+    return this.lmdb.getSync(key);
+  }
+  async put(key, value) {
+    const buffer = typeof value === 'string' ? Buffer.from(value) : value;
+    await this.lmdb.put(key, buffer);
+  }
+  resetReadTxn() {}
+}
+exports.LmdbWrapper = LmdbWrapper;
+function open(directory
+// eslint-disable-next-line no-unused-vars
+) {
+  return new LmdbWrapper(new (_rust().Lmdb)({
+    path: directory,
+    asyncWrites: true,
+    mapSize: 1024 * 1024 * 1024 * 15
+  }));
+}
+const pipeline = (0, _util().promisify)(_stream().default.pipeline);
+class LMDBLiteCache {
+  // $FlowFixMe
+
+  constructor(cacheDir) {
+    this.fs = new (_fs().NodeFS)();
+    this.dir = cacheDir;
+    this.fsCache = new _FSCache.FSCache(this.fs, cacheDir);
+    this.store = open(cacheDir, {
+      name: 'parcel-cache',
+      encoding: 'binary',
+      compression: true
+    });
+  }
+  ensure() {
+    return Promise.resolve();
+  }
+  serialize() {
+    return {
+      dir: this.dir
+    };
+  }
+  static deserialize(opts) {
+    return new LMDBLiteCache(opts.dir);
+  }
+  has(key) {
+    return Promise.resolve(this.store.get(key) != null);
+  }
+  get(key) {
+    let data = this.store.get(key);
+    if (data == null) {
+      return Promise.resolve(null);
+    }
+    return Promise.resolve((0, _core().deserialize)(data));
+  }
+  async set(key, value) {
+    await this.setBlob(key, (0, _core().serialize)(value));
+  }
+  getStream(key) {
+    return this.fs.createReadStream(_path().default.join(this.dir, key));
+  }
+  setStream(key, stream) {
+    return pipeline(stream, this.fs.createWriteStream(_path().default.join(this.dir, key)));
+  }
+  getBlob(key) {
+    try {
+      return Promise.resolve(this.getBlobSync(key));
+    } catch (err) {
+      return Promise.reject(err);
+    }
+  }
+  getBlobSync(key) {
+    const buffer = this.store.get(key);
+    if (buffer == null) {
+      throw new Error(`Key ${key} not found in cache`);
+    }
+    return buffer;
+  }
+  async setBlob(key, contents) {
+    await this.store.put(key, contents);
+  }
+  getBuffer(key) {
+    return Promise.resolve(this.store.get(key));
+  }
+  #getFilePath(key, index) {
+    return _path().default.join(this.dir, `${key}-${index}`);
+  }
+  hasLargeBlob(key) {
+    return this.fs.exists(this.#getFilePath(key, 0));
+  }
+
+  // eslint-disable-next-line require-await
+  async getLargeBlob(key) {
+    return this.fsCache.getLargeBlob(key);
+  }
+
+  // eslint-disable-next-line require-await
+  async setLargeBlob(key, contents, options) {
+    return this.fsCache.setLargeBlob(key, contents, options);
+  }
+  deleteLargeBlob(key) {
+    return this.fsCache.deleteLargeBlob(key);
+  }
+  refresh() {
+    // Reset the read transaction for the store. This guarantees that
+    // the next read will see the latest changes to the store.
+    // Useful in scenarios where reads and writes are multi-threaded.
+    // See https://github.com/kriszyp/lmdb-js#resetreadtxn-void
+    this.store.resetReadTxn();
+  }
+}
+exports.LMDBLiteCache = LMDBLiteCache;
+(0, _core().registerSerializableClass)(`${_package.default.version}:LMDBLiteCache`, LMDBLiteCache);

package/lib/index.js

@@ -14,6 +14,17 @@ Object.keys(_LMDBCache).forEach(function (key) {
     }
   });
 });
+var _LMDBLiteCache = require("./LMDBLiteCache");
+Object.keys(_LMDBLiteCache).forEach(function (key) {
+  if (key === "default" || key === "__esModule") return;
+  if (key in exports && exports[key] === _LMDBLiteCache[key]) return;
+  Object.defineProperty(exports, key, {
+    enumerable: true,
+    get: function () {
+      return _LMDBLiteCache[key];
+    }
+  });
+});
 var _FSCache = require("./FSCache");
 Object.keys(_FSCache).forEach(function (key) {
   if (key === "default" || key === "__esModule") return;

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@atlaspack/cache",
   "description": "Interface for defining caches and file-system, IDB and LMDB implementations.",
-  "version": "2.12.1-dev.3398+81c73b3cd",
-  "license": "MIT",
+  "version": "2.12.1-dev.3443+d1170cfc7",
+  "license": "(MIT OR Apache-2.0)",
   "publishConfig": {
     "access": "public"
   },
@@ -17,17 +17,19 @@
     "node": ">= 16.0.0"
   },
   "scripts": {
+    "test": "mocha",
     "build-ts": "mkdir -p lib && flow-to-ts src/types.js > lib/types.d.ts",
     "check-ts": "tsc --noEmit index.d.ts"
   },
   "dependencies": {
-    "@atlaspack/fs": "2.12.1-dev.3398+81c73b3cd",
-    "@atlaspack/logger": "2.12.1-dev.3398+81c73b3cd",
-    "@atlaspack/utils": "2.12.1-dev.3398+81c73b3cd",
+    "@atlaspack/fs": "2.12.1-dev.3443+d1170cfc7",
+    "@atlaspack/logger": "2.12.1-dev.3443+d1170cfc7",
+    "@atlaspack/rust": "2.12.1-dev.3443+d1170cfc7",
+    "@atlaspack/utils": "2.12.1-dev.3443+d1170cfc7",
     "lmdb": "2.8.5"
   },
   "peerDependencies": {
-    "@atlaspack/core": "^2.12.1-dev.3398+81c73b3cd"
+    "@atlaspack/core": "^2.12.1-dev.3443+d1170cfc7"
   },
   "devDependencies": {
     "idb": "^5.0.8"
@@ -36,5 +38,5 @@
     "./src/IDBCache.js": "./src/IDBCache.browser.js",
     "./src/LMDBCache.js": false
   },
-  "gitHead": "81c73b3cdf93adf8b0013be9fed5422579bd5910"
+  "gitHead": "d1170cfc79beb290b2a066f472f68f71f7d7cb23"
 }

package/src/LMDBLiteCache.js

@@ -0,0 +1,196 @@
+// @flow strict-local
+
+import {Lmdb} from '@atlaspack/rust';
+import type {FilePath} from '@atlaspack/types';
+import type {Cache} from './types';
+import type {Readable, Writable} from 'stream';
+
+import stream from 'stream';
+import path from 'path';
+import {promisify} from 'util';
+import {
+  serialize,
+  deserialize,
+  registerSerializableClass,
+} from '@atlaspack/core';
+import {NodeFS} from '@atlaspack/fs';
+// $FlowFixMe
+import packageJson from '../package.json';
+
+import {FSCache} from './FSCache';
+
+interface DBOpenOptions {
+  name: string;
+  // unused
+  encoding: string;
+  // unused
+  compression: boolean;
+}
+
+export class LmdbWrapper {
+  lmdb: Lmdb;
+
+  constructor(lmdb: Lmdb) {
+    this.lmdb = lmdb;
+
+    // $FlowFixMe
+    this[Symbol.dispose] = () => {
+      this.lmdb.close();
+    };
+  }
+
+  get(key: string): Buffer | null {
+    return this.lmdb.getSync(key);
+  }
+
+  async put(key: string, value: Buffer | string): Promise<void> {
+    const buffer: Buffer =
+      typeof value === 'string' ? Buffer.from(value) : value;
+    await this.lmdb.put(key, buffer);
+  }
+
+  resetReadTxn() {}
+}
+
+export function open(
+  directory: string,
+  // eslint-disable-next-line no-unused-vars
+  openOptions: DBOpenOptions,
+): LmdbWrapper {
+  return new LmdbWrapper(
+    new Lmdb({
+      path: directory,
+      asyncWrites: true,
+      mapSize: 1024 * 1024 * 1024 * 15,
+    }),
+  );
+}
+
+const pipeline: (Readable, Writable) => Promise<void> = promisify(
+  stream.pipeline,
+);
+
+export class LMDBLiteCache implements Cache {
+  fs: NodeFS;
+  dir: FilePath;
+  // $FlowFixMe
+  store: any;
+  fsCache: FSCache;
+
+  constructor(cacheDir: FilePath) {
+    this.fs = new NodeFS();
+    this.dir = cacheDir;
+    this.fsCache = new FSCache(this.fs, cacheDir);
+
+    this.store = open(cacheDir, {
+      name: 'parcel-cache',
+      encoding: 'binary',
+      compression: true,
+    });
+  }
+
+  ensure(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  serialize(): {|dir: FilePath|} {
+    return {
+      dir: this.dir,
+    };
+  }
+
+  static deserialize(opts: {|dir: FilePath|}): LMDBLiteCache {
+    return new LMDBLiteCache(opts.dir);
+  }
+
+  has(key: string): Promise<boolean> {
+    return Promise.resolve(this.store.get(key) != null);
+  }
+
+  get<T>(key: string): Promise<?T> {
+    let data = this.store.get(key);
+    if (data == null) {
+      return Promise.resolve(null);
+    }
+
+    return Promise.resolve(deserialize(data));
+  }
+
+  async set(key: string, value: mixed): Promise<void> {
+    await this.setBlob(key, serialize(value));
+  }
+
+  getStream(key: string): Readable {
+    return this.fs.createReadStream(path.join(this.dir, key));
+  }
+
+  setStream(key: string, stream: Readable): Promise<void> {
+    return pipeline(
+      stream,
+      this.fs.createWriteStream(path.join(this.dir, key)),
+    );
+  }
+
+  getBlob(key: string): Promise<Buffer> {
+    try {
+      return Promise.resolve(this.getBlobSync(key));
+    } catch (err) {
+      return Promise.reject(err);
+    }
+  }
+
+  getBlobSync(key: string): Buffer {
+    const buffer = this.store.get(key);
+    if (buffer == null) {
+      throw new Error(`Key ${key} not found in cache`);
+    }
+    return buffer;
+  }
+
+  async setBlob(key: string, contents: Buffer | string): Promise<void> {
+    await this.store.put(key, contents);
+  }
+
+  getBuffer(key: string): Promise<?Buffer> {
+    return Promise.resolve(this.store.get(key));
+  }
+
+  #getFilePath(key: string, index: number): string {
+    return path.join(this.dir, `${key}-${index}`);
+  }
+
+  hasLargeBlob(key: string): Promise<boolean> {
+    return this.fs.exists(this.#getFilePath(key, 0));
+  }
+
+  // eslint-disable-next-line require-await
+  async getLargeBlob(key: string): Promise<Buffer> {
+    return this.fsCache.getLargeBlob(key);
+  }
+
+  // eslint-disable-next-line require-await
+  async setLargeBlob(
+    key: string,
+    contents: Buffer | string,
+    options?: {|signal?: AbortSignal|},
+  ): Promise<void> {
+    return this.fsCache.setLargeBlob(key, contents, options);
+  }
+
+  deleteLargeBlob(key: string): Promise<void> {
+    return this.fsCache.deleteLargeBlob(key);
+  }
+
+  refresh(): void {
+    // Reset the read transaction for the store. This guarantees that
+    // the next read will see the latest changes to the store.
+    // Useful in scenarios where reads and writes are multi-threaded.
+    // See https://github.com/kriszyp/lmdb-js#resetreadtxn-void
+    this.store.resetReadTxn();
+  }
+}
+
+registerSerializableClass(
+  `${packageJson.version}:LMDBLiteCache`,
+  LMDBLiteCache,
+);

package/src/index.js

@@ -1,5 +1,6 @@
 // @flow
 export type {Cache} from './types';
 export * from './LMDBCache';
+export * from './LMDBLiteCache';
 export * from './FSCache';
 export * from './IDBCache';

package/test/LMDBLiteCache.test.js

@@ -0,0 +1,32 @@
+import * as path from 'node:path';
+import {tmpdir} from 'os';
+import {LMDBLiteCache} from '../src/index';
+import {deserialize, serialize} from 'node:v8';
+import assert from 'node:assert';
+
+const cacheDir = path.join(tmpdir(), 'lmdb-lite-cache-tests');
+
+describe('LMDBLiteCache', () => {
+  it('can be constructed', async () => {
+    const cache = new LMDBLiteCache(cacheDir);
+    await cache.ensure();
+  });
+
+  it('can retrieve keys', async () => {
+    const cache = new LMDBLiteCache(cacheDir);
+    await cache.ensure();
+    await cache.setBlob('key', Buffer.from(serialize({value: 42})));
+    const buffer = await cache.getBlob('key');
+    const result = deserialize(buffer);
+    assert.equal(result.value, 42);
+  });
+
+  it('can retrieve keys synchronously', async () => {
+    const cache = new LMDBLiteCache(cacheDir);
+    await cache.ensure();
+    cache.setBlob('key', Buffer.from(serialize({value: 42})));
+    const buffer = cache.getBlobSync('key');
+    const result = deserialize(buffer);
+    assert.equal(result.value, 42);
+  });
+});