@atlaspack/cache 3.1.1-canary.36 → 3.1.1-canary.361

package/dist/IDBCache.browser.js

@@ -0,0 +1,123 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IDBCache = void 0;
+const stream_1 = require("stream");
+const build_cache_1 = require("@atlaspack/build-cache");
+const utils_1 = require("@atlaspack/utils");
+const idb_1 = require("idb");
+const package_json_1 = __importDefault(require("../package.json"));
+const STORE_NAME = 'cache';
+class IDBCache {
+    constructor() {
+        this.store = (0, idb_1.openDB)('REPL-parcel-cache', 1, {
+            upgrade(db) {
+                db.createObjectStore(STORE_NAME);
+            },
+            blocked() { },
+            blocking() { },
+            terminated() { },
+        });
+    }
+    ensure() {
+        return Promise.resolve();
+    }
+    serialize() {
+        return {
+        /*::...null*/
+        };
+    }
+    static deserialize() {
+        return new IDBCache();
+    }
+    has(key) {
+        return Promise.resolve(this.store.get(key) != null);
+    }
+    async get(key) {
+        let data = await (await this.store).get(STORE_NAME, key);
+        if (data == null) {
+            return null;
+        }
+        return Promise.resolve((0, build_cache_1.deserialize)(data));
+    }
+    async set(key, value) {
+        await (await this.store).put(STORE_NAME, (0, build_cache_1.serialize)(value), key);
+    }
+    getStream(key) {
+        let dataPromise = this.store
+            // @ts-expect-error TS7006
+            .then((s) => s.get(STORE_NAME, key))
+            // @ts-expect-error TS7006
+            .then((d) => Buffer.from(d))
+            // @ts-expect-error TS7006
+            .catch((e) => e);
+        const stream = new stream_1.Readable({
+            async read() {
+                let data = await dataPromise;
+                if (data instanceof Error) {
+                    stream.emit('error', data);
+                }
+                else {
+                    stream.push(Buffer.from(data));
+                    stream.push(null);
+                }
+            },
+        });
+        return stream;
+    }
+    async setStream(key, stream) {
+        let buf = await (0, utils_1.bufferStream)(stream);
+        await (await this.store).put(STORE_NAME, buf, key);
+    }
+    async getBlob(key) {
+        let data = await (await this.store).get(STORE_NAME, key);
+        if (data == null) {
+            return Promise.reject(new Error(`Key ${key} not found in cache`));
+        }
+        return Buffer.from(data.buffer);
+    }
+    async setBlob(key, contents) {
+        let data = contents instanceof Uint8Array ? contents : Buffer.from(contents);
+        await (await this.store).put(STORE_NAME, data, key);
+    }
+    // async setBlobs(
+    //   entries: $ReadOnlyArray<[string, Buffer | string]>,
+    // ): Promise<void> {
+    //   const tx = (await this.store).transaction(STORE_NAME, 'readwrite');
+    //   await Promise.all([
+    //     ...entries.map(([key, value]) =>
+    //       tx.store.put(
+    //         value instanceof Uint8Array ? value : Buffer.from(value),
+    //         key,
+    //       ),
+    //     ),
+    //     tx.done,
+    //   ]);
+    // }
+    async getBuffer(key) {
+        let data = await (await this.store).get(STORE_NAME, key);
+        if (data == null) {
+            return null;
+        }
+        return Buffer.from(data.buffer);
+    }
+    hasLargeBlob(key) {
+        return this.has(key);
+    }
+    getLargeBlob(key) {
+        return this.getBlob(key);
+    }
+    setLargeBlob(key, contents) {
+        return this.setBlob(key, contents);
+    }
+    async deleteLargeBlob(key) {
+        await (await this.store).delete(STORE_NAME, key);
+    }
+    refresh() {
+        // NOOP
+    }
+}
+exports.IDBCache = IDBCache;
+(0, build_cache_1.registerSerializableClass)(`${package_json_1.default.version}:IDBCache`, IDBCache);

package/dist/IDBCache.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IDBCache = void 0;
+// @ts-expect-error TS2420
+class IDBCache {
+    constructor() {
+        throw new Error('IDBCache is only supported in the browser');
+    }
+}
+exports.IDBCache = IDBCache;

package/dist/LMDBLiteCache.js

@@ -0,0 +1,223 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LMDBLiteCache = exports.LmdbWrapper = void 0;
+exports.open = open;
+const build_cache_1 = require("@atlaspack/build-cache");
+const feature_flags_1 = require("@atlaspack/feature-flags");
+const rust_1 = require("@atlaspack/rust");
+// @ts-expect-error TS7016
+const ncp_1 = __importDefault(require("ncp"));
+const util_1 = require("util");
+const stream_1 = __importDefault(require("stream"));
+const path_1 = __importDefault(require("path"));
+const fs_1 = require("@atlaspack/fs");
+const package_json_1 = __importDefault(require("../package.json"));
+const FSCache_1 = require("./FSCache");
+const logger_1 = require("@atlaspack/logger");
+const ncpAsync = (0, util_1.promisify)(ncp_1.default);
+class LmdbWrapper {
+    constructor(lmdb) {
+        this.lmdb = lmdb;
+    }
+    has(key) {
+        return this.lmdb.hasSync(key);
+    }
+    async delete(key) {
+        await this.lmdb.delete(key);
+    }
+    get(key) {
+        return this.lmdb.getSync(key);
+    }
+    async put(key, value) {
+        const buffer = typeof value === 'string' ? Buffer.from(value) : value;
+        await this.lmdb.put(key, buffer);
+    }
+    *keys() {
+        const PAGE_SIZE = 10000000;
+        let currentKeys = this.lmdb.keysSync(0, PAGE_SIZE);
+        while (currentKeys.length > 0) {
+            for (const key of currentKeys) {
+                yield key;
+            }
+            currentKeys = this.lmdb.keysSync(currentKeys.length, PAGE_SIZE);
+        }
+    }
+    compact(targetPath) {
+        this.lmdb.compact(targetPath);
+    }
+}
+exports.LmdbWrapper = LmdbWrapper;
+function open(directory, 
+// eslint-disable-next-line no-unused-vars
+openOptions) {
+    return new LmdbWrapper(new rust_1.Lmdb({
+        path: directory,
+        asyncWrites: true,
+        mapSize: process.env.ATLASPACK_BUILD_ENV === 'test'
+            ? 1024 * 1024 * 1024
+            : 1024 * 1024 * 1024 * 15,
+    }));
+}
+const pipeline = (0, util_1.promisify)(stream_1.default.pipeline);
+class LMDBLiteCache {
+    constructor(cacheDir) {
+        this.fs = new fs_1.NodeFS();
+        this.dir = cacheDir;
+        this.cacheFilesDirectory = path_1.default.join(cacheDir, 'files');
+        this.fsCache = new FSCache_1.FSCache(this.fs, cacheDir);
+        this.store = open(cacheDir, {
+            name: 'parcel-cache',
+            encoding: 'binary',
+            compression: true,
+        });
+    }
+    /**
+     * Use this to pass the native LMDB instance back to Rust.
+     */
+    getNativeRef() {
+        return this.store.lmdb;
+    }
+    async ensure() {
+        if (!(0, feature_flags_1.getFeatureFlag)('cachePerformanceImprovements')) {
+            await this.fsCache.ensure();
+        }
+        await this.fs.mkdirp(this.cacheFilesDirectory);
+        return Promise.resolve();
+    }
+    serialize() {
+        return {
+            dir: this.dir,
+        };
+    }
+    static deserialize(cache) {
+        return new LMDBLiteCache(cache.dir);
+    }
+    has(key) {
+        return Promise.resolve(this.store.has(key));
+    }
+    get(key) {
+        let data = this.store.get(key);
+        if (data == null) {
+            return Promise.resolve(null);
+        }
+        return Promise.resolve((0, build_cache_1.deserialize)(data));
+    }
+    async set(key, value) {
+        await this.setBlob(key, (0, build_cache_1.serialize)(value));
+    }
+    getStream(key) {
+        if (!(0, feature_flags_1.getFeatureFlag)('cachePerformanceImprovements')) {
+            return this.fs.createReadStream(path_1.default.join(this.dir, key));
+        }
+        return this.fs.createReadStream(this.getFileKey(key));
+    }
+    async setStream(key, stream) {
+        if (!(0, feature_flags_1.getFeatureFlag)('cachePerformanceImprovements')) {
+            return pipeline(stream, this.fs.createWriteStream(path_1.default.join(this.dir, key)));
+        }
+        const filePath = this.getFileKey(key);
+        await this.fs.mkdirp(path_1.default.dirname(filePath));
+        return pipeline(stream, this.fs.createWriteStream(filePath));
+    }
+    // eslint-disable-next-line require-await
+    async getBlob(key) {
+        return this.getBlobSync(key);
+    }
+    getBlobSync(key) {
+        const buffer = this.store.get(key);
+        if (buffer == null) {
+            throw new Error(`Key ${key} not found in cache`);
+        }
+        return buffer;
+    }
+    async setBlob(key, contents) {
+        await this.store.put(key, contents);
+    }
+    getBuffer(key) {
+        return Promise.resolve(this.store.get(key));
+    }
+    hasLargeBlob(key) {
+        if (!(0, feature_flags_1.getFeatureFlag)('cachePerformanceImprovements')) {
+            return this.fsCache.hasLargeBlob(key);
+        }
+        return this.fs.exists(this.getFileKey(key));
+    }
+    getLargeBlob(key) {
+        if (!(0, feature_flags_1.getFeatureFlag)('cachePerformanceImprovements')) {
+            return this.fsCache.getLargeBlob(key);
+        }
+        return this.fs.readFile(this.getFileKey(key));
+    }
+    async setLargeBlob(key, contents, options) {
+        if (!(0, feature_flags_1.getFeatureFlag)('cachePerformanceImprovements')) {
+            return this.fsCache.setLargeBlob(key, contents, options);
+        }
+        const targetPath = this.getFileKey(key);
+        await this.fs.mkdirp(path_1.default.dirname(targetPath));
+        return this.fs.writeFile(targetPath, contents);
+    }
+    /**
+     * @deprecated Use store.delete instead.
+     */
+    deleteLargeBlob(key) {
+        if (!(0, feature_flags_1.getFeatureFlag)('cachePerformanceImprovements')) {
+            return this.fsCache.deleteLargeBlob(key);
+        }
+        return this.store.delete(key);
+    }
+    keys() {
+        return this.store.keys();
+    }
+    async compact(targetPath) {
+        await this.fs.mkdirp(targetPath);
+        const files = await this.fs.readdir(this.dir);
+        // copy all files except data.mdb and lock.mdb to the target path (recursive)
+        for (const file of files) {
+            const filePath = path_1.default.join(this.dir, file);
+            if (file === 'data.mdb' || file === 'lock.mdb') {
+                continue;
+            }
+            await ncpAsync(filePath, path_1.default.join(targetPath, file));
+        }
+        this.store.compact(path_1.default.join(targetPath, 'data.mdb'));
+    }
+    refresh() { }
+    /**
+     * Streams, packages are stored in files instead of LMDB.
+     *
+     * On this case, if a cache key happens to have a parent traversal, ../..
+     * it is treated specially
+     *
+     * That is, something/../something and something are meant to be different
+     * keys.
+     *
+     * Plus we do not want to store values outside of the cache directory.
+     */
+    getFileKey(key) {
+        const cleanKey = key
+            .split('/')
+            .map((part) => {
+            if (part === '..') {
+                return '$$__parent_dir$$';
+            }
+            return part;
+        })
+            .join('/');
+        return path_1.default.join(this.cacheFilesDirectory, cleanKey);
+    }
+    async clear() {
+        await (0, logger_1.instrumentAsync)('LMDBLiteCache::clear', async () => {
+            const keys = await this.keys();
+            for (const key of keys) {
+                await this.store.delete(key);
+            }
+            await this.fs.rimraf(this.cacheFilesDirectory);
+            await this.fs.mkdirp(this.cacheFilesDirectory);
+        });
+    }
+}
+exports.LMDBLiteCache = LMDBLiteCache;
+(0, build_cache_1.registerSerializableClass)(`${package_json_1.default.version}:LMDBLiteCache`, LMDBLiteCache);

package/dist/constants.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WRITE_LIMIT_CHUNK = void 0;
+// Node has a file size limit of 2 GB
+exports.WRITE_LIMIT_CHUNK = 2 * 1024 ** 3;

package/dist/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./FSCache"), exports);
+__exportStar(require("./IDBCache"), exports);
+__exportStar(require("./LMDBLiteCache"), exports);

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/lib/FSCache.js

@@ -25,6 +25,20 @@ function _util() {
   };
   return data;
 }
+function _rust() {
+  const data = require("@atlaspack/rust");
+  _rust = function () {
+    return data;
+  };
+  return data;
+}
+function _featureFlags() {
+  const data = require("@atlaspack/feature-flags");
+  _featureFlags = function () {
+    return data;
+  };
+  return data;
+}
 function _logger() {
   const data = _interopRequireDefault(require("@atlaspack/logger"));
   _logger = function () {
@@ -43,6 +57,7 @@ var _package = _interopRequireDefault(require("../package.json"));
 var _constants = require("./constants");
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 // flowlint-next-line untyped-import:off
+
 const pipeline = (0, _util().promisify)(_stream().default.pipeline);
 class FSCache {
   constructor(fs, cacheDir) {
@@ -57,11 +72,17 @@ class FSCache {
     // This speeds up large caches on many file systems since there are fewer files in a single directory.
     let dirPromises = [];
     for (let i = 0; i < 256; i++) {
-      dirPromises.push(this.fs.mkdirp(_path().default.join(this.dir, ('00' + i.toString(16)).slice(-2))));
+      dirPromises.push(
+      // @ts-expect-error TS2345
+      this.fs.mkdirp(_path().default.join(this.dir, ('00' + i.toString(16)).slice(-2))));
     }
     await Promise.all(dirPromises);
   }
   _getCachePath(cacheId) {
+    if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
+      const cleanId = (0, _rust().hashString)(cacheId);
+      return _path().default.join(this.dir, cleanId.slice(0, 2), cleanId.slice(2));
+    }
     return _path().default.join(this.dir, cacheId.slice(0, 2), cacheId.slice(2));
   }
   getStream(key) {
@@ -91,6 +112,9 @@ class FSCache {
     }
   }
   #getFilePath(key, index) {
+    if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
+      return _path().default.join(this.dir, `${(0, _rust().hashString)(key)}-${index}`);
+    }
     return _path().default.join(this.dir, `${key}-${index}`);
   }
   async #unlinkChunks(key, index) {
@@ -117,12 +141,19 @@ class FSCache {
     const writePromises = [];
     if (chunks === 1) {
       // If there's one chunk, don't slice the content
-      writePromises.push(this.fs.writeFile(this.#getFilePath(key, 0), contents, {
+      writePromises.push(
+      // @ts-expect-error TS2345
+      this.fs.writeFile(this.#getFilePath(key, 0), contents, {
+        // @ts-expect-error TS2353
         signal: options === null || options === void 0 ? void 0 : options.signal
       }));
     } else {
       for (let i = 0; i < chunks; i += 1) {
-        writePromises.push(this.fs.writeFile(this.#getFilePath(key, i), typeof contents === 'string' ? contents.slice(i * _constants.WRITE_LIMIT_CHUNK, (i + 1) * _constants.WRITE_LIMIT_CHUNK) : contents.subarray(i * _constants.WRITE_LIMIT_CHUNK, (i + 1) * _constants.WRITE_LIMIT_CHUNK), {
+        writePromises.push(
+        // @ts-expect-error TS2345
+        this.fs.writeFile(this.#getFilePath(key, i), typeof contents === 'string' ? contents.slice(i * _constants.WRITE_LIMIT_CHUNK, (i + 1) * _constants.WRITE_LIMIT_CHUNK) : contents.subarray(i * _constants.WRITE_LIMIT_CHUNK, (i + 1) * _constants.WRITE_LIMIT_CHUNK),
+        // @ts-expect-error TS2353
+        {
           signal: options === null || options === void 0 ? void 0 : options.signal
         }));
       }
@@ -137,6 +168,7 @@ class FSCache {
     let i = 0;
     let filePath = this.#getFilePath(key, i);
     while (await this.fs.exists(filePath)) {
+      // @ts-expect-error TS2345
       deletePromises.push(this.fs.rimraf(filePath));
       i += 1;
       filePath = this.#getFilePath(key, i);

package/lib/IDBCache.browser.js

@@ -34,12 +34,8 @@ function _idb() {
 }
 var _package = _interopRequireDefault(require("../package.json"));
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-// $FlowFixMe[untyped-import]
-// $FlowFixMe[untyped-import]
 const STORE_NAME = 'cache';
 class IDBCache {
-  // $FlowFixMe
-
   constructor() {
     this.store = (0, _idb().openDB)('REPL-parcel-cache', 1, {
       upgrade(db) {
@@ -75,9 +71,14 @@ class IDBCache {
     await (await this.store).put(STORE_NAME, (0, _buildCache().serialize)(value), key);
   }
   getStream(key) {
-    let dataPromise = this.store.then(s => s.get(STORE_NAME, key)).then(d => Buffer.from(d)).catch(e => e);
+    let dataPromise = this.store
+    // @ts-expect-error TS7006
+    .then(s => s.get(STORE_NAME, key))
+    // @ts-expect-error TS7006
+    .then(d => Buffer.from(d))
+    // @ts-expect-error TS7006
+    .catch(e => e);
     const stream = new (_stream().Readable)({
-      // $FlowFixMe(incompatible-call)
       async read() {
         let data = await dataPromise;
         if (data instanceof Error) {

package/lib/IDBCache.js

@@ -4,7 +4,7 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.IDBCache = void 0;
-// $FlowFixMe
+// @ts-expect-error TS2420
 class IDBCache {
   constructor() {
     throw new Error('IDBCache is only supported in the browser');