@atlaspack/cache 3.1.1-canary.48 → 3.1.1-canary.481
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +602 -0
- package/dist/FSCache.js +162 -0
- package/dist/IDBCache.browser.js +124 -0
- package/dist/IDBCache.js +10 -0
- package/dist/LMDBLiteCache.js +198 -0
- package/dist/constants.js +5 -0
- package/dist/index.js +19 -0
- package/dist/types.js +2 -0
- package/lib/FSCache.js +15 -25
- package/lib/IDBCache.browser.js +10 -7
- package/lib/IDBCache.js +1 -1
- package/lib/LMDBLiteCache.js +20 -63
- package/lib/types/FSCache.d.ts +27 -0
- package/lib/types/IDBCache.browser.d.ts +22 -0
- package/lib/types/IDBCache.d.ts +4 -0
- package/lib/types/LMDBLiteCache.d.ts +78 -0
- package/lib/types/constants.d.ts +1 -0
- package/lib/types/index.d.ts +4 -0
- package/lib/types/types.d.ts +2 -0
- package/package.json +14 -15
- package/src/{FSCache.js → FSCache.ts} +21 -24
- package/src/{IDBCache.browser.js → IDBCache.browser.ts} +9 -10
- package/src/{IDBCache.js → IDBCache.ts} +1 -2
- package/src/{LMDBLiteCache.js → LMDBLiteCache.ts} +26 -60
- package/src/{constants.js → constants.ts} +0 -2
- package/src/{index.js → index.ts} +0 -2
- package/src/{types.js → types.ts} +0 -1
- package/test/{LMDBLiteCache.test.js → LMDBLiteCache.test.ts} +9 -11
- package/tsconfig.json +27 -0
- package/tsconfig.tsbuildinfo +1 -0
- package/index.d.ts +0 -12
- package/lib/types.d.ts +0 -2
package/dist/FSCache.js
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
|
|
3
|
+
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
|
|
4
|
+
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
|
5
|
+
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
|
6
|
+
};
|
|
7
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
8
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
9
|
+
};
|
|
10
|
+
var _FSCache_instances, _FSCache_getFilePath, _FSCache_unlinkChunks;
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.FSCache = void 0;
|
|
13
|
+
const stream_1 = __importDefault(require("stream"));
|
|
14
|
+
const path_1 = __importDefault(require("path"));
|
|
15
|
+
const util_1 = require("util");
|
|
16
|
+
const logger_1 = __importDefault(require("@atlaspack/logger"));
|
|
17
|
+
const build_cache_1 = require("@atlaspack/build-cache");
|
|
18
|
+
// flowlint-next-line untyped-import:off
|
|
19
|
+
const package_json_1 = __importDefault(require("../package.json"));
|
|
20
|
+
const constants_1 = require("./constants");
|
|
21
|
+
const pipeline = (0, util_1.promisify)(stream_1.default.pipeline);
|
|
22
|
+
class FSCache {
|
|
23
|
+
constructor(fs, cacheDir) {
|
|
24
|
+
_FSCache_instances.add(this);
|
|
25
|
+
this.fs = fs;
|
|
26
|
+
this.dir = cacheDir;
|
|
27
|
+
}
|
|
28
|
+
async ensure() {
|
|
29
|
+
// First, create the main cache directory if necessary.
|
|
30
|
+
await this.fs.mkdirp(this.dir);
|
|
31
|
+
// In parallel, create sub-directories for every possible hex value
|
|
32
|
+
// This speeds up large caches on many file systems since there are fewer files in a single directory.
|
|
33
|
+
let dirPromises = [];
|
|
34
|
+
for (let i = 0; i < 256; i++) {
|
|
35
|
+
dirPromises.push(
|
|
36
|
+
// @ts-expect-error TS2345
|
|
37
|
+
this.fs.mkdirp(path_1.default.join(this.dir, ('00' + i.toString(16)).slice(-2))));
|
|
38
|
+
}
|
|
39
|
+
await Promise.all(dirPromises);
|
|
40
|
+
}
|
|
41
|
+
_getCachePath(cacheId) {
|
|
42
|
+
return path_1.default.join(this.dir, cacheId.slice(0, 2), cacheId.slice(2));
|
|
43
|
+
}
|
|
44
|
+
getStream(key) {
|
|
45
|
+
return this.fs.createReadStream(this._getCachePath(`${key}-large`));
|
|
46
|
+
}
|
|
47
|
+
setStream(key, stream) {
|
|
48
|
+
return pipeline(stream, this.fs.createWriteStream(this._getCachePath(`${key}-large`)));
|
|
49
|
+
}
|
|
50
|
+
has(key) {
|
|
51
|
+
return this.fs.exists(this._getCachePath(key));
|
|
52
|
+
}
|
|
53
|
+
getBlob(key) {
|
|
54
|
+
return this.fs.readFile(this._getCachePath(key));
|
|
55
|
+
}
|
|
56
|
+
async setBlob(key, contents) {
|
|
57
|
+
await this.fs.writeFile(this._getCachePath(key), contents);
|
|
58
|
+
}
|
|
59
|
+
async getBuffer(key) {
|
|
60
|
+
try {
|
|
61
|
+
return await this.fs.readFile(this._getCachePath(key));
|
|
62
|
+
}
|
|
63
|
+
catch (err) {
|
|
64
|
+
if (err.code === 'ENOENT') {
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
throw err;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
hasLargeBlob(key) {
|
|
73
|
+
return this.fs.exists(__classPrivateFieldGet(this, _FSCache_instances, "m", _FSCache_getFilePath).call(this, key, 0));
|
|
74
|
+
}
|
|
75
|
+
async getLargeBlob(key) {
|
|
76
|
+
const buffers = [];
|
|
77
|
+
for (let i = 0; await this.fs.exists(__classPrivateFieldGet(this, _FSCache_instances, "m", _FSCache_getFilePath).call(this, key, i)); i += 1) {
|
|
78
|
+
const file = this.fs.readFile(__classPrivateFieldGet(this, _FSCache_instances, "m", _FSCache_getFilePath).call(this, key, i));
|
|
79
|
+
buffers.push(file);
|
|
80
|
+
}
|
|
81
|
+
return Buffer.concat(await Promise.all(buffers));
|
|
82
|
+
}
|
|
83
|
+
async setLargeBlob(key, contents, options) {
|
|
84
|
+
const chunks = Math.ceil(contents.length / constants_1.WRITE_LIMIT_CHUNK);
|
|
85
|
+
const writePromises = [];
|
|
86
|
+
if (chunks === 1) {
|
|
87
|
+
// If there's one chunk, don't slice the content
|
|
88
|
+
writePromises.push(
|
|
89
|
+
// @ts-expect-error TS2345
|
|
90
|
+
this.fs.writeFile(__classPrivateFieldGet(this, _FSCache_instances, "m", _FSCache_getFilePath).call(this, key, 0), contents, {
|
|
91
|
+
// @ts-expect-error TS2353
|
|
92
|
+
signal: options?.signal,
|
|
93
|
+
}));
|
|
94
|
+
}
|
|
95
|
+
else {
|
|
96
|
+
for (let i = 0; i < chunks; i += 1) {
|
|
97
|
+
writePromises.push(
|
|
98
|
+
// @ts-expect-error TS2345
|
|
99
|
+
this.fs.writeFile(__classPrivateFieldGet(this, _FSCache_instances, "m", _FSCache_getFilePath).call(this, key, i), typeof contents === 'string'
|
|
100
|
+
? contents.slice(i * constants_1.WRITE_LIMIT_CHUNK, (i + 1) * constants_1.WRITE_LIMIT_CHUNK)
|
|
101
|
+
: contents.subarray(i * constants_1.WRITE_LIMIT_CHUNK, (i + 1) * constants_1.WRITE_LIMIT_CHUNK),
|
|
102
|
+
// @ts-expect-error TS2353
|
|
103
|
+
{ signal: options?.signal }));
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
// If there's already a files following this chunk, it's old and should be removed
|
|
107
|
+
writePromises.push(__classPrivateFieldGet(this, _FSCache_instances, "m", _FSCache_unlinkChunks).call(this, key, chunks));
|
|
108
|
+
await Promise.all(writePromises);
|
|
109
|
+
}
|
|
110
|
+
async deleteLargeBlob(key) {
|
|
111
|
+
const deletePromises = [];
|
|
112
|
+
let i = 0;
|
|
113
|
+
let filePath = __classPrivateFieldGet(this, _FSCache_instances, "m", _FSCache_getFilePath).call(this, key, i);
|
|
114
|
+
while (await this.fs.exists(filePath)) {
|
|
115
|
+
// @ts-expect-error TS2345
|
|
116
|
+
deletePromises.push(this.fs.rimraf(filePath));
|
|
117
|
+
i += 1;
|
|
118
|
+
filePath = __classPrivateFieldGet(this, _FSCache_instances, "m", _FSCache_getFilePath).call(this, key, i);
|
|
119
|
+
}
|
|
120
|
+
await Promise.all(deletePromises);
|
|
121
|
+
}
|
|
122
|
+
async get(key) {
|
|
123
|
+
try {
|
|
124
|
+
let data = await this.fs.readFile(this._getCachePath(key));
|
|
125
|
+
return (0, build_cache_1.deserialize)(data);
|
|
126
|
+
}
|
|
127
|
+
catch (err) {
|
|
128
|
+
if (err.code === 'ENOENT') {
|
|
129
|
+
return null;
|
|
130
|
+
}
|
|
131
|
+
else {
|
|
132
|
+
throw err;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
async set(key, value) {
|
|
137
|
+
try {
|
|
138
|
+
let blobPath = this._getCachePath(key);
|
|
139
|
+
let data = (0, build_cache_1.serialize)(value);
|
|
140
|
+
await this.fs.writeFile(blobPath, data);
|
|
141
|
+
}
|
|
142
|
+
catch (err) {
|
|
143
|
+
logger_1.default.error(err, '@atlaspack/cache');
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
refresh() {
|
|
147
|
+
// NOOP
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
exports.FSCache = FSCache;
|
|
151
|
+
_FSCache_instances = new WeakSet(), _FSCache_getFilePath = function _FSCache_getFilePath(key, index) {
|
|
152
|
+
return path_1.default.join(this.dir, `${key}-${index}`);
|
|
153
|
+
}, _FSCache_unlinkChunks = async function _FSCache_unlinkChunks(key, index) {
|
|
154
|
+
try {
|
|
155
|
+
await this.fs.unlink(__classPrivateFieldGet(this, _FSCache_instances, "m", _FSCache_getFilePath).call(this, key, index));
|
|
156
|
+
await __classPrivateFieldGet(this, _FSCache_instances, "m", _FSCache_unlinkChunks).call(this, key, index + 1);
|
|
157
|
+
}
|
|
158
|
+
catch (err) {
|
|
159
|
+
// If there's an error, no more chunks are left to delete
|
|
160
|
+
}
|
|
161
|
+
};
|
|
162
|
+
(0, build_cache_1.registerSerializableClass)(`${package_json_1.default.version}:FSCache`, FSCache);
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.IDBCache = void 0;
|
|
7
|
+
const stream_1 = require("stream");
|
|
8
|
+
const build_cache_1 = require("@atlaspack/build-cache");
|
|
9
|
+
const utils_1 = require("@atlaspack/utils");
|
|
10
|
+
// @ts-expect-error TS1479: idb is ESM-only but we need to import it in this browser-specific CJS file
|
|
11
|
+
const idb_1 = require("idb");
|
|
12
|
+
const package_json_1 = __importDefault(require("../package.json"));
|
|
13
|
+
const STORE_NAME = 'cache';
|
|
14
|
+
class IDBCache {
|
|
15
|
+
constructor() {
|
|
16
|
+
this.store = (0, idb_1.openDB)('REPL-parcel-cache', 1, {
|
|
17
|
+
upgrade(db) {
|
|
18
|
+
db.createObjectStore(STORE_NAME);
|
|
19
|
+
},
|
|
20
|
+
blocked() { },
|
|
21
|
+
blocking() { },
|
|
22
|
+
terminated() { },
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
ensure() {
|
|
26
|
+
return Promise.resolve();
|
|
27
|
+
}
|
|
28
|
+
serialize() {
|
|
29
|
+
return {
|
|
30
|
+
/*::...null*/
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
static deserialize() {
|
|
34
|
+
return new IDBCache();
|
|
35
|
+
}
|
|
36
|
+
has(key) {
|
|
37
|
+
return Promise.resolve(this.store.get(key) != null);
|
|
38
|
+
}
|
|
39
|
+
async get(key) {
|
|
40
|
+
let data = await (await this.store).get(STORE_NAME, key);
|
|
41
|
+
if (data == null) {
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
return Promise.resolve((0, build_cache_1.deserialize)(data));
|
|
45
|
+
}
|
|
46
|
+
async set(key, value) {
|
|
47
|
+
await (await this.store).put(STORE_NAME, (0, build_cache_1.serialize)(value), key);
|
|
48
|
+
}
|
|
49
|
+
getStream(key) {
|
|
50
|
+
let dataPromise = this.store
|
|
51
|
+
// @ts-expect-error TS7006
|
|
52
|
+
.then((s) => s.get(STORE_NAME, key))
|
|
53
|
+
// @ts-expect-error TS7006
|
|
54
|
+
.then((d) => Buffer.from(d))
|
|
55
|
+
// @ts-expect-error TS7006
|
|
56
|
+
.catch((e) => e);
|
|
57
|
+
const stream = new stream_1.Readable({
|
|
58
|
+
async read() {
|
|
59
|
+
let data = await dataPromise;
|
|
60
|
+
if (data instanceof Error) {
|
|
61
|
+
stream.emit('error', data);
|
|
62
|
+
}
|
|
63
|
+
else {
|
|
64
|
+
stream.push(Buffer.from(data));
|
|
65
|
+
stream.push(null);
|
|
66
|
+
}
|
|
67
|
+
},
|
|
68
|
+
});
|
|
69
|
+
return stream;
|
|
70
|
+
}
|
|
71
|
+
async setStream(key, stream) {
|
|
72
|
+
let buf = await (0, utils_1.bufferStream)(stream);
|
|
73
|
+
await (await this.store).put(STORE_NAME, buf, key);
|
|
74
|
+
}
|
|
75
|
+
async getBlob(key) {
|
|
76
|
+
let data = await (await this.store).get(STORE_NAME, key);
|
|
77
|
+
if (data == null) {
|
|
78
|
+
return Promise.reject(new Error(`Key ${key} not found in cache`));
|
|
79
|
+
}
|
|
80
|
+
return Buffer.from(data.buffer);
|
|
81
|
+
}
|
|
82
|
+
async setBlob(key, contents) {
|
|
83
|
+
let data = contents instanceof Uint8Array ? contents : Buffer.from(contents);
|
|
84
|
+
await (await this.store).put(STORE_NAME, data, key);
|
|
85
|
+
}
|
|
86
|
+
// async setBlobs(
|
|
87
|
+
// entries: $ReadOnlyArray<[string, Buffer | string]>,
|
|
88
|
+
// ): Promise<void> {
|
|
89
|
+
// const tx = (await this.store).transaction(STORE_NAME, 'readwrite');
|
|
90
|
+
// await Promise.all([
|
|
91
|
+
// ...entries.map(([key, value]) =>
|
|
92
|
+
// tx.store.put(
|
|
93
|
+
// value instanceof Uint8Array ? value : Buffer.from(value),
|
|
94
|
+
// key,
|
|
95
|
+
// ),
|
|
96
|
+
// ),
|
|
97
|
+
// tx.done,
|
|
98
|
+
// ]);
|
|
99
|
+
// }
|
|
100
|
+
async getBuffer(key) {
|
|
101
|
+
let data = await (await this.store).get(STORE_NAME, key);
|
|
102
|
+
if (data == null) {
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
return Buffer.from(data.buffer);
|
|
106
|
+
}
|
|
107
|
+
hasLargeBlob(key) {
|
|
108
|
+
return this.has(key);
|
|
109
|
+
}
|
|
110
|
+
getLargeBlob(key) {
|
|
111
|
+
return this.getBlob(key);
|
|
112
|
+
}
|
|
113
|
+
setLargeBlob(key, contents) {
|
|
114
|
+
return this.setBlob(key, contents);
|
|
115
|
+
}
|
|
116
|
+
async deleteLargeBlob(key) {
|
|
117
|
+
await (await this.store).delete(STORE_NAME, key);
|
|
118
|
+
}
|
|
119
|
+
refresh() {
|
|
120
|
+
// NOOP
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
exports.IDBCache = IDBCache;
|
|
124
|
+
(0, build_cache_1.registerSerializableClass)(`${package_json_1.default.version}:IDBCache`, IDBCache);
|
package/dist/IDBCache.js
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.IDBCache = void 0;
|
|
4
|
+
// @ts-expect-error TS2420
|
|
5
|
+
class IDBCache {
|
|
6
|
+
constructor() {
|
|
7
|
+
throw new Error('IDBCache is only supported in the browser');
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
exports.IDBCache = IDBCache;
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.LMDBLiteCache = exports.LmdbWrapper = void 0;
|
|
7
|
+
exports.open = open;
|
|
8
|
+
const build_cache_1 = require("@atlaspack/build-cache");
|
|
9
|
+
const rust_1 = require("@atlaspack/rust");
|
|
10
|
+
// @ts-expect-error TS7016
|
|
11
|
+
const ncp_1 = __importDefault(require("ncp"));
|
|
12
|
+
const util_1 = require("util");
|
|
13
|
+
const stream_1 = __importDefault(require("stream"));
|
|
14
|
+
const path_1 = __importDefault(require("path"));
|
|
15
|
+
const fs_1 = require("@atlaspack/fs");
|
|
16
|
+
const package_json_1 = __importDefault(require("../package.json"));
|
|
17
|
+
const FSCache_1 = require("./FSCache");
|
|
18
|
+
const logger_1 = require("@atlaspack/logger");
|
|
19
|
+
const ncpAsync = (0, util_1.promisify)(ncp_1.default);
|
|
20
|
+
class LmdbWrapper {
|
|
21
|
+
constructor(lmdb) {
|
|
22
|
+
this.lmdb = lmdb;
|
|
23
|
+
}
|
|
24
|
+
has(key) {
|
|
25
|
+
return this.lmdb.hasSync(key);
|
|
26
|
+
}
|
|
27
|
+
async delete(key) {
|
|
28
|
+
await this.lmdb.delete(key);
|
|
29
|
+
}
|
|
30
|
+
get(key) {
|
|
31
|
+
return this.lmdb.getSync(key);
|
|
32
|
+
}
|
|
33
|
+
async put(key, value) {
|
|
34
|
+
const buffer = typeof value === 'string' ? Buffer.from(value) : value;
|
|
35
|
+
await this.lmdb.put(key, buffer);
|
|
36
|
+
}
|
|
37
|
+
*keys() {
|
|
38
|
+
const PAGE_SIZE = 10000000;
|
|
39
|
+
let currentKeys = this.lmdb.keysSync(0, PAGE_SIZE);
|
|
40
|
+
while (currentKeys.length > 0) {
|
|
41
|
+
for (const key of currentKeys) {
|
|
42
|
+
yield key;
|
|
43
|
+
}
|
|
44
|
+
currentKeys = this.lmdb.keysSync(currentKeys.length, PAGE_SIZE);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
compact(targetPath) {
|
|
48
|
+
this.lmdb.compact(targetPath);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
exports.LmdbWrapper = LmdbWrapper;
|
|
52
|
+
function open(directory,
|
|
53
|
+
// eslint-disable-next-line no-unused-vars
|
|
54
|
+
openOptions) {
|
|
55
|
+
return new LmdbWrapper(new rust_1.Lmdb({
|
|
56
|
+
path: directory,
|
|
57
|
+
asyncWrites: true,
|
|
58
|
+
mapSize: process.env.ATLASPACK_BUILD_ENV === 'test'
|
|
59
|
+
? 256 * 1024 * 1024
|
|
60
|
+
: 1024 * 1024 * 1024 * 15,
|
|
61
|
+
}));
|
|
62
|
+
}
|
|
63
|
+
const pipeline = (0, util_1.promisify)(stream_1.default.pipeline);
|
|
64
|
+
class LMDBLiteCache {
|
|
65
|
+
constructor(cacheDir) {
|
|
66
|
+
this.fs = new fs_1.NodeFS();
|
|
67
|
+
this.dir = cacheDir;
|
|
68
|
+
this.cacheFilesDirectory = path_1.default.join(cacheDir, 'files');
|
|
69
|
+
this.fsCache = new FSCache_1.FSCache(this.fs, cacheDir);
|
|
70
|
+
this.store = open(cacheDir, {
|
|
71
|
+
name: 'parcel-cache',
|
|
72
|
+
encoding: 'binary',
|
|
73
|
+
compression: true,
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Use this to pass the native LMDB instance back to Rust.
|
|
78
|
+
*/
|
|
79
|
+
getNativeRef() {
|
|
80
|
+
return this.store.lmdb;
|
|
81
|
+
}
|
|
82
|
+
async ensure() {
|
|
83
|
+
await this.fsCache.ensure();
|
|
84
|
+
await this.fs.mkdirp(this.cacheFilesDirectory);
|
|
85
|
+
return Promise.resolve();
|
|
86
|
+
}
|
|
87
|
+
serialize() {
|
|
88
|
+
return {
|
|
89
|
+
dir: this.dir,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
static deserialize(cache) {
|
|
93
|
+
return new LMDBLiteCache(cache.dir);
|
|
94
|
+
}
|
|
95
|
+
has(key) {
|
|
96
|
+
return Promise.resolve(this.store.has(key));
|
|
97
|
+
}
|
|
98
|
+
get(key) {
|
|
99
|
+
let data = this.store.get(key);
|
|
100
|
+
if (data == null) {
|
|
101
|
+
return Promise.resolve(null);
|
|
102
|
+
}
|
|
103
|
+
return Promise.resolve((0, build_cache_1.deserialize)(data));
|
|
104
|
+
}
|
|
105
|
+
async set(key, value) {
|
|
106
|
+
await this.setBlob(key, (0, build_cache_1.serialize)(value));
|
|
107
|
+
}
|
|
108
|
+
getStream(key) {
|
|
109
|
+
return this.fs.createReadStream(path_1.default.join(this.dir, key));
|
|
110
|
+
}
|
|
111
|
+
setStream(key, stream) {
|
|
112
|
+
return pipeline(stream, this.fs.createWriteStream(path_1.default.join(this.dir, key)));
|
|
113
|
+
}
|
|
114
|
+
// eslint-disable-next-line require-await
|
|
115
|
+
async getBlob(key) {
|
|
116
|
+
return this.getBlobSync(key);
|
|
117
|
+
}
|
|
118
|
+
getBlobSync(key) {
|
|
119
|
+
const buffer = this.store.get(key);
|
|
120
|
+
if (buffer == null) {
|
|
121
|
+
throw new Error(`Key ${key} not found in cache`);
|
|
122
|
+
}
|
|
123
|
+
return buffer;
|
|
124
|
+
}
|
|
125
|
+
async setBlob(key, contents) {
|
|
126
|
+
await this.store.put(key, contents);
|
|
127
|
+
}
|
|
128
|
+
getBuffer(key) {
|
|
129
|
+
return Promise.resolve(this.store.get(key));
|
|
130
|
+
}
|
|
131
|
+
hasLargeBlob(key) {
|
|
132
|
+
return this.fsCache.hasLargeBlob(key);
|
|
133
|
+
}
|
|
134
|
+
getLargeBlob(key) {
|
|
135
|
+
return this.fsCache.getLargeBlob(key);
|
|
136
|
+
}
|
|
137
|
+
setLargeBlob(key, contents, options) {
|
|
138
|
+
return this.fsCache.setLargeBlob(key, contents, options);
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* @deprecated Use store.delete instead.
|
|
142
|
+
*/
|
|
143
|
+
deleteLargeBlob(key) {
|
|
144
|
+
return this.fsCache.deleteLargeBlob(key);
|
|
145
|
+
}
|
|
146
|
+
keys() {
|
|
147
|
+
return this.store.keys();
|
|
148
|
+
}
|
|
149
|
+
async compact(targetPath) {
|
|
150
|
+
await this.fs.mkdirp(targetPath);
|
|
151
|
+
const files = await this.fs.readdir(this.dir);
|
|
152
|
+
// copy all files except data.mdb and lock.mdb to the target path (recursive)
|
|
153
|
+
for (const file of files) {
|
|
154
|
+
const filePath = path_1.default.join(this.dir, file);
|
|
155
|
+
if (file === 'data.mdb' || file === 'lock.mdb') {
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
await ncpAsync(filePath, path_1.default.join(targetPath, file));
|
|
159
|
+
}
|
|
160
|
+
this.store.compact(path_1.default.join(targetPath, 'data.mdb'));
|
|
161
|
+
}
|
|
162
|
+
refresh() { }
|
|
163
|
+
/**
|
|
164
|
+
* Streams, packages are stored in files instead of LMDB.
|
|
165
|
+
*
|
|
166
|
+
* On this case, if a cache key happens to have a parent traversal, ../..
|
|
167
|
+
* it is treated specially
|
|
168
|
+
*
|
|
169
|
+
* That is, something/../something and something are meant to be different
|
|
170
|
+
* keys.
|
|
171
|
+
*
|
|
172
|
+
* Plus we do not want to store values outside of the cache directory.
|
|
173
|
+
*/
|
|
174
|
+
getFileKey(key) {
|
|
175
|
+
const cleanKey = key
|
|
176
|
+
.split('/')
|
|
177
|
+
.map((part) => {
|
|
178
|
+
if (part === '..') {
|
|
179
|
+
return '$$__parent_dir$$';
|
|
180
|
+
}
|
|
181
|
+
return part;
|
|
182
|
+
})
|
|
183
|
+
.join('/');
|
|
184
|
+
return path_1.default.join(this.cacheFilesDirectory, cleanKey);
|
|
185
|
+
}
|
|
186
|
+
async clear() {
|
|
187
|
+
await (0, logger_1.instrumentAsync)('LMDBLiteCache::clear', async () => {
|
|
188
|
+
const keys = await this.keys();
|
|
189
|
+
for (const key of keys) {
|
|
190
|
+
await this.store.delete(key);
|
|
191
|
+
}
|
|
192
|
+
await this.fs.rimraf(this.cacheFilesDirectory);
|
|
193
|
+
await this.fs.mkdirp(this.cacheFilesDirectory);
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
exports.LMDBLiteCache = LMDBLiteCache;
|
|
198
|
+
(0, build_cache_1.registerSerializableClass)(`${package_json_1.default.version}:LMDBLiteCache`, LMDBLiteCache);
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./FSCache"), exports);
|
|
18
|
+
__exportStar(require("./IDBCache"), exports);
|
|
19
|
+
__exportStar(require("./LMDBLiteCache"), exports);
|
package/dist/types.js
ADDED
package/lib/FSCache.js
CHANGED
|
@@ -25,20 +25,6 @@ function _util() {
|
|
|
25
25
|
};
|
|
26
26
|
return data;
|
|
27
27
|
}
|
|
28
|
-
function _rust() {
|
|
29
|
-
const data = require("@atlaspack/rust");
|
|
30
|
-
_rust = function () {
|
|
31
|
-
return data;
|
|
32
|
-
};
|
|
33
|
-
return data;
|
|
34
|
-
}
|
|
35
|
-
function _featureFlags() {
|
|
36
|
-
const data = require("@atlaspack/feature-flags");
|
|
37
|
-
_featureFlags = function () {
|
|
38
|
-
return data;
|
|
39
|
-
};
|
|
40
|
-
return data;
|
|
41
|
-
}
|
|
42
28
|
function _logger() {
|
|
43
29
|
const data = _interopRequireDefault(require("@atlaspack/logger"));
|
|
44
30
|
_logger = function () {
|
|
@@ -55,8 +41,9 @@ function _buildCache() {
|
|
|
55
41
|
}
|
|
56
42
|
var _package = _interopRequireDefault(require("../package.json"));
|
|
57
43
|
var _constants = require("./constants");
|
|
58
|
-
function _interopRequireDefault(
|
|
44
|
+
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
59
45
|
// flowlint-next-line untyped-import:off
|
|
46
|
+
|
|
60
47
|
const pipeline = (0, _util().promisify)(_stream().default.pipeline);
|
|
61
48
|
class FSCache {
|
|
62
49
|
constructor(fs, cacheDir) {
|
|
@@ -71,15 +58,13 @@ class FSCache {
|
|
|
71
58
|
// This speeds up large caches on many file systems since there are fewer files in a single directory.
|
|
72
59
|
let dirPromises = [];
|
|
73
60
|
for (let i = 0; i < 256; i++) {
|
|
74
|
-
dirPromises.push(
|
|
61
|
+
dirPromises.push(
|
|
62
|
+
// @ts-expect-error TS2345
|
|
63
|
+
this.fs.mkdirp(_path().default.join(this.dir, ('00' + i.toString(16)).slice(-2))));
|
|
75
64
|
}
|
|
76
65
|
await Promise.all(dirPromises);
|
|
77
66
|
}
|
|
78
67
|
_getCachePath(cacheId) {
|
|
79
|
-
if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
|
80
|
-
const cleanId = (0, _rust().hashString)(cacheId);
|
|
81
|
-
return _path().default.join(this.dir, cleanId.slice(0, 2), cleanId.slice(2));
|
|
82
|
-
}
|
|
83
68
|
return _path().default.join(this.dir, cacheId.slice(0, 2), cacheId.slice(2));
|
|
84
69
|
}
|
|
85
70
|
getStream(key) {
|
|
@@ -109,9 +94,6 @@ class FSCache {
|
|
|
109
94
|
}
|
|
110
95
|
}
|
|
111
96
|
#getFilePath(key, index) {
|
|
112
|
-
if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
|
113
|
-
return _path().default.join(this.dir, `${(0, _rust().hashString)(key)}-${index}`);
|
|
114
|
-
}
|
|
115
97
|
return _path().default.join(this.dir, `${key}-${index}`);
|
|
116
98
|
}
|
|
117
99
|
async #unlinkChunks(key, index) {
|
|
@@ -138,12 +120,19 @@ class FSCache {
|
|
|
138
120
|
const writePromises = [];
|
|
139
121
|
if (chunks === 1) {
|
|
140
122
|
// If there's one chunk, don't slice the content
|
|
141
|
-
writePromises.push(
|
|
123
|
+
writePromises.push(
|
|
124
|
+
// @ts-expect-error TS2345
|
|
125
|
+
this.fs.writeFile(this.#getFilePath(key, 0), contents, {
|
|
126
|
+
// @ts-expect-error TS2353
|
|
142
127
|
signal: options === null || options === void 0 ? void 0 : options.signal
|
|
143
128
|
}));
|
|
144
129
|
} else {
|
|
145
130
|
for (let i = 0; i < chunks; i += 1) {
|
|
146
|
-
writePromises.push(
|
|
131
|
+
writePromises.push(
|
|
132
|
+
// @ts-expect-error TS2345
|
|
133
|
+
this.fs.writeFile(this.#getFilePath(key, i), typeof contents === 'string' ? contents.slice(i * _constants.WRITE_LIMIT_CHUNK, (i + 1) * _constants.WRITE_LIMIT_CHUNK) : contents.subarray(i * _constants.WRITE_LIMIT_CHUNK, (i + 1) * _constants.WRITE_LIMIT_CHUNK),
|
|
134
|
+
// @ts-expect-error TS2353
|
|
135
|
+
{
|
|
147
136
|
signal: options === null || options === void 0 ? void 0 : options.signal
|
|
148
137
|
}));
|
|
149
138
|
}
|
|
@@ -158,6 +147,7 @@ class FSCache {
|
|
|
158
147
|
let i = 0;
|
|
159
148
|
let filePath = this.#getFilePath(key, i);
|
|
160
149
|
while (await this.fs.exists(filePath)) {
|
|
150
|
+
// @ts-expect-error TS2345
|
|
161
151
|
deletePromises.push(this.fs.rimraf(filePath));
|
|
162
152
|
i += 1;
|
|
163
153
|
filePath = this.#getFilePath(key, i);
|
package/lib/IDBCache.browser.js
CHANGED
|
@@ -33,13 +33,11 @@ function _idb() {
|
|
|
33
33
|
return data;
|
|
34
34
|
}
|
|
35
35
|
var _package = _interopRequireDefault(require("../package.json"));
|
|
36
|
-
function _interopRequireDefault(
|
|
37
|
-
//
|
|
38
|
-
|
|
36
|
+
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
37
|
+
// @ts-expect-error TS1479: idb is ESM-only but we need to import it in this browser-specific CJS file
|
|
38
|
+
|
|
39
39
|
const STORE_NAME = 'cache';
|
|
40
40
|
class IDBCache {
|
|
41
|
-
// $FlowFixMe
|
|
42
|
-
|
|
43
41
|
constructor() {
|
|
44
42
|
this.store = (0, _idb().openDB)('REPL-parcel-cache', 1, {
|
|
45
43
|
upgrade(db) {
|
|
@@ -75,9 +73,14 @@ class IDBCache {
|
|
|
75
73
|
await (await this.store).put(STORE_NAME, (0, _buildCache().serialize)(value), key);
|
|
76
74
|
}
|
|
77
75
|
getStream(key) {
|
|
78
|
-
let dataPromise = this.store
|
|
76
|
+
let dataPromise = this.store
|
|
77
|
+
// @ts-expect-error TS7006
|
|
78
|
+
.then(s => s.get(STORE_NAME, key))
|
|
79
|
+
// @ts-expect-error TS7006
|
|
80
|
+
.then(d => Buffer.from(d))
|
|
81
|
+
// @ts-expect-error TS7006
|
|
82
|
+
.catch(e => e);
|
|
79
83
|
const stream = new (_stream().Readable)({
|
|
80
|
-
// $FlowFixMe(incompatible-call)
|
|
81
84
|
async read() {
|
|
82
85
|
let data = await dataPromise;
|
|
83
86
|
if (data instanceof Error) {
|
package/lib/IDBCache.js
CHANGED