@atlaspack/cache 3.1.1-canary.36 → 3.1.1-canary.360

package/lib/LMDBLiteCache.js

@@ -26,13 +26,6 @@ function _rust() {
   };
   return data;
 }
-function _fs() {
-  const data = _interopRequireDefault(require("fs"));
-  _fs = function () {
-    return data;
-  };
-  return data;
-}
 function _ncp() {
   const data = _interopRequireDefault(require("ncp"));
   _ncp = function () {
@@ -61,26 +54,29 @@ function _path() {
   };
   return data;
 }
-function _fs2() {
+function _fs() {
   const data = require("@atlaspack/fs");
-  _fs2 = function () {
+  _fs = function () {
     return data;
   };
   return data;
 }
 var _package = _interopRequireDefault(require("../package.json"));
 var _FSCache = require("./FSCache");
+function _logger() {
+  const data = require("@atlaspack/logger");
+  _logger = function () {
+    return data;
+  };
+  return data;
+}
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-// $FlowFixMe
+// @ts-expect-error TS7016
+
 const ncpAsync = (0, _util().promisify)(_ncp().default);
 class LmdbWrapper {
   constructor(lmdb) {
     this.lmdb = lmdb;
-
-    // $FlowFixMe
-    this[Symbol.dispose] = () => {
-      this.lmdb.close();
-    };
   }
   has(key) {
     return this.lmdb.hasSync(key);
@@ -110,20 +106,25 @@ class LmdbWrapper {
   }
 }
 exports.LmdbWrapper = LmdbWrapper;
-function open(directory
+function open(directory,
 // eslint-disable-next-line no-unused-vars
-) {
+openOptions) {
   return new LmdbWrapper(new (_rust().Lmdb)({
     path: directory,
     asyncWrites: true,
-    mapSize: 1024 * 1024 * 1024 * 15
+    mapSize: process.env.ATLASPACK_BUILD_ENV === 'test' ? 1024 * 1024 * 1024 : 1024 * 1024 * 1024 * 15
   }));
 }
 const pipeline = (0, _util().promisify)(_stream().default.pipeline);
 class LMDBLiteCache {
+  /**
+   * Directory where we store raw files.
+   */
+
   constructor(cacheDir) {
-    this.fs = new (_fs2().NodeFS)();
+    this.fs = new (_fs().NodeFS)();
     this.dir = cacheDir;
+    this.cacheFilesDirectory = _path().default.join(cacheDir, 'files');
     this.fsCache = new _FSCache.FSCache(this.fs, cacheDir);
     this.store = open(cacheDir, {
       name: 'parcel-cache',
@@ -142,6 +143,7 @@ class LMDBLiteCache {
     if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
       await this.fsCache.ensure();
     }
+    await this.fs.mkdirp(this.cacheFilesDirectory);
     return Promise.resolve();
   }
   serialize() {
@@ -166,10 +168,18 @@ class LMDBLiteCache {
     await this.setBlob(key, (0, _buildCache().serialize)(value));
   }
   getStream(key) {
-    return this.fs.createReadStream(_path().default.join(this.dir, key));
+    if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
+      return this.fs.createReadStream(_path().default.join(this.dir, key));
+    }
+    return this.fs.createReadStream(this.getFileKey(key));
   }
-  setStream(key, stream) {
-    return pipeline(stream, this.fs.createWriteStream(_path().default.join(this.dir, key)));
+  async setStream(key, stream) {
+    if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
+      return pipeline(stream, this.fs.createWriteStream(_path().default.join(this.dir, key)));
+    }
+    const filePath = this.getFileKey(key);
+    await this.fs.mkdirp(_path().default.dirname(filePath));
+    return pipeline(stream, this.fs.createWriteStream(filePath));
   }
 
   // eslint-disable-next-line require-await
@@ -189,34 +199,25 @@ class LMDBLiteCache {
   getBuffer(key) {
     return Promise.resolve(this.store.get(key));
   }
-  #getFilePath(key, index) {
-    return _path().default.join(this.dir, `${key}-${index}`);
-  }
   hasLargeBlob(key) {
     if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
       return this.fsCache.hasLargeBlob(key);
     }
-    return this.has(key);
+    return this.fs.exists(this.getFileKey(key));
   }
-
-  /**
-   * @deprecated Use getBlob instead.
-   */
   getLargeBlob(key) {
     if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
       return this.fsCache.getLargeBlob(key);
     }
-    return Promise.resolve(this.getBlobSync(key));
+    return this.fs.readFile(this.getFileKey(key));
   }
-
-  /**
-   * @deprecated Use setBlob instead.
-   */
-  setLargeBlob(key, contents, options) {
+  async setLargeBlob(key, contents, options) {
     if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
       return this.fsCache.setLargeBlob(key, contents, options);
     }
-    return this.setBlob(key, contents);
+    const targetPath = this.getFileKey(key);
+    await this.fs.mkdirp(_path().default.dirname(targetPath));
+    return this.fs.writeFile(targetPath, contents);
   }
 
   /**
@@ -232,10 +233,8 @@ class LMDBLiteCache {
     return this.store.keys();
   }
   async compact(targetPath) {
-    await _fs().default.promises.mkdir(targetPath, {
-      recursive: true
-    });
-    const files = await _fs().default.promises.readdir(this.dir);
+    await this.fs.mkdirp(targetPath);
+    const files = await this.fs.readdir(this.dir);
     // copy all files except data.mdb and lock.mdb to the target path (recursive)
     for (const file of files) {
       const filePath = _path().default.join(this.dir, file);
@@ -247,6 +246,37 @@ class LMDBLiteCache {
     this.store.compact(_path().default.join(targetPath, 'data.mdb'));
   }
   refresh() {}
+
+  /**
+   * Streams, packages are stored in files instead of LMDB.
+   *
+   * On this case, if a cache key happens to have a parent traversal, ../..
+   * it is treated specially
+   *
+   * That is, something/../something and something are meant to be different
+   * keys.
+   *
+   * Plus we do not want to store values outside of the cache directory.
+   */
+  getFileKey(key) {
+    const cleanKey = key.split('/').map(part => {
+      if (part === '..') {
+        return '$$__parent_dir$$';
+      }
+      return part;
+    }).join('/');
+    return _path().default.join(this.cacheFilesDirectory, cleanKey);
+  }
+  async clear() {
+    await (0, _logger().instrumentAsync)('LMDBLiteCache::clear', async () => {
+      const keys = await this.keys();
+      for (const key of keys) {
+        await this.store.delete(key);
+      }
+      await this.fs.rimraf(this.cacheFilesDirectory);
+      await this.fs.mkdirp(this.cacheFilesDirectory);
+    });
+  }
 }
 exports.LMDBLiteCache = LMDBLiteCache;
 (0, _buildCache().registerSerializableClass)(`${_package.default.version}:LMDBLiteCache`, LMDBLiteCache);

package/lib/types/FSCache.d.ts

@@ -0,0 +1,27 @@
+import type { Readable } from 'stream';
+import type { FilePath } from '@atlaspack/types';
+import type { FileSystem } from '@atlaspack/fs';
+import type { Cache } from './types';
+export declare class FSCache implements Cache {
+    #private;
+    fs: FileSystem;
+    dir: FilePath;
+    constructor(fs: FileSystem, cacheDir: FilePath);
+    ensure(): Promise<void>;
+    _getCachePath(cacheId: string): FilePath;
+    getStream(key: string): Readable;
+    setStream(key: string, stream: Readable): Promise<void>;
+    has(key: string): Promise<boolean>;
+    getBlob(key: string): Promise<Buffer>;
+    setBlob(key: string, contents: Buffer | string): Promise<void>;
+    getBuffer(key: string): Promise<Buffer | null | undefined>;
+    hasLargeBlob(key: string): Promise<boolean>;
+    getLargeBlob(key: string): Promise<Buffer>;
+    setLargeBlob(key: string, contents: Buffer | string, options?: {
+        signal?: AbortSignal;
+    }): Promise<void>;
+    deleteLargeBlob(key: string): Promise<void>;
+    get<T>(key: string): Promise<T | null | undefined>;
+    set(key: string, value: unknown): Promise<void>;
+    refresh(): void;
+}

package/lib/types/IDBCache.browser.d.ts

@@ -0,0 +1,22 @@
+import type { Cache } from './types';
+import { Readable } from 'stream';
+export declare class IDBCache implements Cache {
+    store: any;
+    constructor();
+    ensure(): Promise<void>;
+    serialize(): Record<any, any>;
+    static deserialize(): IDBCache;
+    has(key: string): Promise<boolean>;
+    get<T>(key: string): Promise<T | null | undefined>;
+    set(key: string, value: unknown): Promise<void>;
+    getStream(key: string): Readable;
+    setStream(key: string, stream: Readable): Promise<void>;
+    getBlob(key: string): Promise<Buffer>;
+    setBlob(key: string, contents: Buffer | string): Promise<void>;
+    getBuffer(key: string): Promise<Buffer | null | undefined>;
+    hasLargeBlob(key: string): Promise<boolean>;
+    getLargeBlob(key: string): Promise<Buffer>;
+    setLargeBlob(key: string, contents: Buffer | string): Promise<void>;
+    deleteLargeBlob(key: string): Promise<void>;
+    refresh(): void;
+}

package/lib/types/IDBCache.d.ts

@@ -0,0 +1,4 @@
+import type { Cache } from './types';
+export declare class IDBCache implements Cache {
+    constructor();
+}

package/lib/types/LMDBLiteCache.d.ts

@@ -0,0 +1,78 @@
+import { Lmdb } from '@atlaspack/rust';
+import type { FilePath } from '@atlaspack/types';
+import type { Cache } from './types';
+import type { Readable } from 'stream';
+import { NodeFS } from '@atlaspack/fs';
+import { FSCache } from './FSCache';
+interface DBOpenOptions {
+    name: string;
+    encoding: string;
+    compression: boolean;
+}
+export declare class LmdbWrapper {
+    lmdb: Lmdb;
+    constructor(lmdb: Lmdb);
+    has(key: string): boolean;
+    delete(key: string): Promise<void>;
+    get(key: string): Buffer | null;
+    put(key: string, value: Buffer | string): Promise<void>;
+    keys(): Iterable<string>;
+    compact(targetPath: string): void;
+}
+export declare function open(directory: string, openOptions: DBOpenOptions): LmdbWrapper;
+export type SerLMDBLiteCache = {
+    dir: FilePath;
+};
+export declare class LMDBLiteCache implements Cache {
+    fs: NodeFS;
+    dir: FilePath;
+    store: LmdbWrapper;
+    fsCache: FSCache;
+    /**
+     * Directory where we store raw files.
+     */
+    cacheFilesDirectory: FilePath;
+    constructor(cacheDir: FilePath);
+    /**
+     * Use this to pass the native LMDB instance back to Rust.
+     */
+    getNativeRef(): Lmdb;
+    ensure(): Promise<void>;
+    serialize(): SerLMDBLiteCache;
+    static deserialize(cache: SerLMDBLiteCache): LMDBLiteCache;
+    has(key: string): Promise<boolean>;
+    get<T>(key: string): Promise<T | null | undefined>;
+    set(key: string, value: unknown): Promise<void>;
+    getStream(key: string): Readable;
+    setStream(key: string, stream: Readable): Promise<void>;
+    getBlob(key: string): Promise<Buffer>;
+    getBlobSync(key: string): Buffer;
+    setBlob(key: string, contents: Buffer | string): Promise<void>;
+    getBuffer(key: string): Promise<Buffer | null | undefined>;
+    hasLargeBlob(key: string): Promise<boolean>;
+    getLargeBlob(key: string): Promise<Buffer>;
+    setLargeBlob(key: string, contents: Buffer | string, options?: {
+        signal?: AbortSignal;
+    }): Promise<void>;
+    /**
+     * @deprecated Use store.delete instead.
+     */
+    deleteLargeBlob(key: string): Promise<void>;
+    keys(): Iterable<string>;
+    compact(targetPath: string): Promise<void>;
+    refresh(): void;
+    /**
+     * Streams, packages are stored in files instead of LMDB.
+     *
+     * On this case, if a cache key happens to have a parent traversal, ../..
+     * it is treated specially
+     *
+     * That is, something/../something and something are meant to be different
+     * keys.
+     *
+     * Plus we do not want to store values outside of the cache directory.
+     */
+    getFileKey(key: string): string;
+    clear(): Promise<void>;
+}
+export {};

package/lib/types/constants.d.ts

@@ -0,0 +1 @@
+export declare const WRITE_LIMIT_CHUNK: number;

package/lib/types/index.d.ts

@@ -0,0 +1,4 @@
+export * from './FSCache';
+export * from './IDBCache';
+export * from './LMDBLiteCache';
+export type { Cache } from './types';

package/lib/types/types.d.ts

@@ -0,0 +1,2 @@
+import type { Cache } from '@atlaspack/types';
+export type { Cache };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@atlaspack/cache",
   "description": "Interface for defining caches and file-system, IDB and LMDB implementations.",
-  "version": "3.1.1-canary.36+2d10c6656",
+  "version": "3.1.1-canary.360+fc3adc098",
   "license": "(MIT OR Apache-2.0)",
   "type": "commonjs",
   "publishConfig": {
@@ -11,24 +11,23 @@
     "type": "git",
     "url": "https://github.com/atlassian-labs/atlaspack.git"
   },
-  "main": "lib/index.js",
-  "source": "src/index.js",
-  "types": "index.d.ts",
+  "main": "./lib/index.js",
+  "source": "./src/index.ts",
+  "types": "./lib/types/index.d.ts",
   "engines": {
     "node": ">= 16.0.0"
   },
   "scripts": {
     "test": "mocha",
-    "build-ts": "mkdir -p lib && flow-to-ts src/types.js > lib/types.d.ts",
-    "check-ts": "tsc --noEmit index.d.ts"
+    "build:lib": "gulp build --gulpfile ../../../gulpfile.js --cwd ."
   },
   "dependencies": {
-    "@atlaspack/build-cache": "2.13.3-canary.104+2d10c6656",
-    "@atlaspack/feature-flags": "2.14.1-canary.104+2d10c6656",
-    "@atlaspack/fs": "2.14.5-canary.36+2d10c6656",
-    "@atlaspack/logger": "2.14.5-canary.36+2d10c6656",
-    "@atlaspack/rust": "3.2.1-canary.36+2d10c6656",
-    "@atlaspack/utils": "2.14.5-canary.36+2d10c6656",
+    "@atlaspack/build-cache": "2.13.3-canary.428+fc3adc098",
+    "@atlaspack/feature-flags": "2.14.1-canary.428+fc3adc098",
+    "@atlaspack/fs": "2.14.5-canary.360+fc3adc098",
+    "@atlaspack/logger": "2.14.5-canary.360+fc3adc098",
+    "@atlaspack/rust": "3.2.1-canary.360+fc3adc098",
+    "@atlaspack/utils": "2.14.5-canary.360+fc3adc098",
     "ncp": "^2.0.0"
   },
   "devDependencies": {
@@ -37,5 +36,5 @@
   "browser": {
     "./src/IDBCache.js": "./src/IDBCache.browser.js"
   },
-  "gitHead": "2d10c6656fc58743c5dbed1d2b6c5666887f9fe4"
-}
+  "gitHead": "fc3adc098f583e40d6d7687412cac6dde7cbb3f3"
+}

package/src/{FSCache.js → FSCache.ts}

@@ -1,5 +1,3 @@
-// @flow strict-local
-
 import type {Readable, Writable} from 'stream';
 import type {FilePath} from '@atlaspack/types';
 import type {FileSystem} from '@atlaspack/fs';
@@ -9,6 +7,8 @@ import stream from 'stream';
 import path from 'path';
 import {promisify} from 'util';
 
+import {hashString} from '@atlaspack/rust';
+import {getFeatureFlag} from '@atlaspack/feature-flags';
 import logger from '@atlaspack/logger';
 import {
   deserialize,
@@ -21,7 +21,7 @@ import packageJson from '../package.json';
 
 import {WRITE_LIMIT_CHUNK} from './constants';
 
-const pipeline: (Readable, Writable) => Promise<void> = promisify(
+const pipeline: (arg1: Readable, arg2: Writable) => Promise<void> = promisify(
   stream.pipeline,
 );
 
@@ -40,9 +40,10 @@ export class FSCache implements Cache {
 
     // In parallel, create sub-directories for every possible hex value
     // This speeds up large caches on many file systems since there are fewer files in a single directory.
-    let dirPromises = [];
+    let dirPromises: Array<Promise<undefined>> = [];
     for (let i = 0; i < 256; i++) {
       dirPromises.push(
+        // @ts-expect-error TS2345
         this.fs.mkdirp(path.join(this.dir, ('00' + i.toString(16)).slice(-2))),
       );
     }
@@ -51,6 +52,10 @@ export class FSCache implements Cache {
   }
 
   _getCachePath(cacheId: string): FilePath {
+    if (getFeatureFlag('cachePerformanceImprovements')) {
+      const cleanId = hashString(cacheId);
+      return path.join(this.dir, cleanId.slice(0, 2), cleanId.slice(2));
+    }
     return path.join(this.dir, cacheId.slice(0, 2), cacheId.slice(2));
   }
 
@@ -77,10 +82,10 @@ export class FSCache implements Cache {
     await this.fs.writeFile(this._getCachePath(key), contents);
   }
 
-  async getBuffer(key: string): Promise<?Buffer> {
+  async getBuffer(key: string): Promise<Buffer | null | undefined> {
     try {
       return await this.fs.readFile(this._getCachePath(key));
-    } catch (err) {
+    } catch (err: any) {
       if (err.code === 'ENOENT') {
         return null;
       } else {
@@ -90,14 +95,17 @@ export class FSCache implements Cache {
   }
 
   #getFilePath(key: string, index: number): string {
+    if (getFeatureFlag('cachePerformanceImprovements')) {
+      return path.join(this.dir, `${hashString(key)}-${index}`);
+    }
     return path.join(this.dir, `${key}-${index}`);
   }
 
-  async #unlinkChunks(key: string, index: number): Promise<void> {
+  async #unlinkChunks(key: string, index: number): Promise<undefined> {
     try {
       await this.fs.unlink(this.#getFilePath(key, index));
       await this.#unlinkChunks(key, index + 1);
-    } catch (err) {
+    } catch (err: any) {
       // If there's an error, no more chunks are left to delete
     }
   }
@@ -120,21 +128,26 @@ export class FSCache implements Cache {
   async setLargeBlob(
     key: string,
     contents: Buffer | string,
-    options?: {|signal?: AbortSignal|},
+    options?: {
+      signal?: AbortSignal;
+    },
   ): Promise<void> {
     const chunks = Math.ceil(contents.length / WRITE_LIMIT_CHUNK);
 
-    const writePromises: Promise<void>[] = [];
+    const writePromises: Promise<undefined>[] = [];
     if (chunks === 1) {
       // If there's one chunk, don't slice the content
       writePromises.push(
+        // @ts-expect-error TS2345
         this.fs.writeFile(this.#getFilePath(key, 0), contents, {
+          // @ts-expect-error TS2353
           signal: options?.signal,
         }),
       );
     } else {
       for (let i = 0; i < chunks; i += 1) {
         writePromises.push(
+          // @ts-expect-error TS2345
           this.fs.writeFile(
             this.#getFilePath(key, i),
             typeof contents === 'string'
@@ -146,6 +159,7 @@ export class FSCache implements Cache {
                   i * WRITE_LIMIT_CHUNK,
                   (i + 1) * WRITE_LIMIT_CHUNK,
                 ),
+            // @ts-expect-error TS2353
             {signal: options?.signal},
           ),
         );
@@ -159,12 +173,13 @@ export class FSCache implements Cache {
   }
 
   async deleteLargeBlob(key: string): Promise<void> {
-    const deletePromises: Promise<void>[] = [];
+    const deletePromises: Promise<undefined>[] = [];
 
     let i = 0;
     let filePath = this.#getFilePath(key, i);
 
     while (await this.fs.exists(filePath)) {
+      // @ts-expect-error TS2345
       deletePromises.push(this.fs.rimraf(filePath));
       i += 1;
       filePath = this.#getFilePath(key, i);
@@ -173,11 +188,11 @@ export class FSCache implements Cache {
     await Promise.all(deletePromises);
   }
 
-  async get<T>(key: string): Promise<?T> {
+  async get<T>(key: string): Promise<T | null | undefined> {
     try {
       let data = await this.fs.readFile(this._getCachePath(key));
       return deserialize(data);
-    } catch (err) {
+    } catch (err: any) {
       if (err.code === 'ENOENT') {
         return null;
       } else {
@@ -186,13 +201,13 @@ export class FSCache implements Cache {
     }
   }
 
-  async set(key: string, value: mixed): Promise<void> {
+  async set(key: string, value: unknown): Promise<void> {
     try {
       let blobPath = this._getCachePath(key);
       let data = serialize(value);
 
       await this.fs.writeFile(blobPath, data);
-    } catch (err) {
+    } catch (err: any) {
       logger.error(err, '@atlaspack/cache');
     }
   }

package/src/{IDBCache.browser.js → IDBCache.browser.ts}

@@ -1,4 +1,3 @@
-// @flow strict-local
 import type {Cache} from './types';
 
 import {Readable} from 'stream';
@@ -9,21 +8,18 @@ import {
   serialize,
 } from '@atlaspack/build-cache';
 import {bufferStream} from '@atlaspack/utils';
-// $FlowFixMe[untyped-import]
 import {openDB} from 'idb';
 
-// $FlowFixMe[untyped-import]
 import packageJson from '../package.json';
 
 const STORE_NAME = 'cache';
 
 export class IDBCache implements Cache {
-  // $FlowFixMe
   store: any;
 
   constructor() {
     this.store = openDB('REPL-parcel-cache', 1, {
-      upgrade(db) {
+      upgrade(db: any) {
         db.createObjectStore(STORE_NAME);
       },
       blocked() {},
@@ -36,7 +32,7 @@ export class IDBCache implements Cache {
     return Promise.resolve();
   }
 
-  serialize(): {||} {
+  serialize(): Record<any, any> {
     return {
       /*::...null*/
     };
@@ -50,7 +46,7 @@ export class IDBCache implements Cache {
     return Promise.resolve(this.store.get(key) != null);
   }
 
-  async get<T>(key: string): Promise<?T> {
+  async get<T>(key: string): Promise<T | null | undefined> {
     let data = await (await this.store).get(STORE_NAME, key);
     if (data == null) {
       return null;
@@ -59,17 +55,19 @@ export class IDBCache implements Cache {
     return Promise.resolve(deserialize(data));
   }
 
-  async set(key: string, value: mixed): Promise<void> {
+  async set(key: string, value: unknown): Promise<void> {
     await (await this.store).put(STORE_NAME, serialize(value), key);
   }
 
   getStream(key: string): Readable {
     let dataPromise = this.store
+      // @ts-expect-error TS7006
       .then((s) => s.get(STORE_NAME, key))
+      // @ts-expect-error TS7006
       .then((d) => Buffer.from(d))
+      // @ts-expect-error TS7006
       .catch((e) => e);
     const stream = new Readable({
-      // $FlowFixMe(incompatible-call)
       async read() {
         let data = await dataPromise;
         if (data instanceof Error) {
@@ -118,7 +116,7 @@ export class IDBCache implements Cache {
   //   ]);
   // }
 
-  async getBuffer(key: string): Promise<?Buffer> {
+  async getBuffer(key: string): Promise<Buffer | null | undefined> {
     let data = await (await this.store).get(STORE_NAME, key);
     if (data == null) {
       return null;

package/src/{IDBCache.js → IDBCache.ts}

@@ -1,7 +1,6 @@
-// @flow strict-local
 import type {Cache} from './types';
 
-// $FlowFixMe
+// @ts-expect-error TS2420
 export class IDBCache implements Cache {
   constructor() {
     throw new Error('IDBCache is only supported in the browser');