@atlaspack/cache 3.1.1-canary.3 → 3.1.1-canary.300
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +379 -0
- package/dist/FSCache.js +171 -0
- package/dist/IDBCache.browser.js +123 -0
- package/dist/IDBCache.js +10 -0
- package/dist/LMDBLiteCache.js +223 -0
- package/dist/constants.js +5 -0
- package/dist/index.js +19 -0
- package/dist/types.js +2 -0
- package/lib/FSCache.js +35 -3
- package/lib/IDBCache.browser.js +7 -6
- package/lib/IDBCache.js +1 -1
- package/lib/LMDBLiteCache.js +110 -41
- package/lib/types/FSCache.d.ts +27 -0
- package/lib/types/IDBCache.browser.d.ts +22 -0
- package/lib/types/IDBCache.d.ts +4 -0
- package/lib/types/LMDBLiteCache.d.ts +78 -0
- package/lib/types/constants.d.ts +1 -0
- package/lib/types/index.d.ts +4 -0
- package/lib/types/types.d.ts +2 -0
- package/package.json +14 -14
- package/src/{FSCache.js → FSCache.ts} +30 -15
- package/src/{IDBCache.browser.js → IDBCache.browser.ts} +8 -10
- package/src/{IDBCache.js → IDBCache.ts} +1 -2
- package/src/{LMDBLiteCache.js → LMDBLiteCache.ts} +119 -48
- package/src/{constants.js → constants.ts} +0 -2
- package/src/{index.js → index.ts} +0 -2
- package/src/{types.js → types.ts} +0 -1
- package/test/LMDBLiteCache.test.ts +241 -0
- package/test/workerThreadsTest.js +42 -0
- package/tsconfig.json +27 -0
- package/tsconfig.tsbuildinfo +1 -0
- package/index.d.ts +0 -12
- package/lib/types.d.ts +0 -2
- package/test/LMDBLiteCache.test.js +0 -33
package/lib/LMDBLiteCache.js
CHANGED
|
@@ -26,6 +26,20 @@ function _rust() {
|
|
|
26
26
|
};
|
|
27
27
|
return data;
|
|
28
28
|
}
|
|
29
|
+
function _ncp() {
|
|
30
|
+
const data = _interopRequireDefault(require("ncp"));
|
|
31
|
+
_ncp = function () {
|
|
32
|
+
return data;
|
|
33
|
+
};
|
|
34
|
+
return data;
|
|
35
|
+
}
|
|
36
|
+
function _util() {
|
|
37
|
+
const data = require("util");
|
|
38
|
+
_util = function () {
|
|
39
|
+
return data;
|
|
40
|
+
};
|
|
41
|
+
return data;
|
|
42
|
+
}
|
|
29
43
|
function _stream() {
|
|
30
44
|
const data = _interopRequireDefault(require("stream"));
|
|
31
45
|
_stream = function () {
|
|
@@ -40,13 +54,6 @@ function _path() {
|
|
|
40
54
|
};
|
|
41
55
|
return data;
|
|
42
56
|
}
|
|
43
|
-
function _util() {
|
|
44
|
-
const data = require("util");
|
|
45
|
-
_util = function () {
|
|
46
|
-
return data;
|
|
47
|
-
};
|
|
48
|
-
return data;
|
|
49
|
-
}
|
|
50
57
|
function _fs() {
|
|
51
58
|
const data = require("@atlaspack/fs");
|
|
52
59
|
_fs = function () {
|
|
@@ -56,16 +63,20 @@ function _fs() {
|
|
|
56
63
|
}
|
|
57
64
|
var _package = _interopRequireDefault(require("../package.json"));
|
|
58
65
|
var _FSCache = require("./FSCache");
|
|
66
|
+
function _logger() {
|
|
67
|
+
const data = require("@atlaspack/logger");
|
|
68
|
+
_logger = function () {
|
|
69
|
+
return data;
|
|
70
|
+
};
|
|
71
|
+
return data;
|
|
72
|
+
}
|
|
59
73
|
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
|
60
|
-
//
|
|
74
|
+
// @ts-expect-error TS7016
|
|
75
|
+
|
|
76
|
+
const ncpAsync = (0, _util().promisify)(_ncp().default);
|
|
61
77
|
class LmdbWrapper {
|
|
62
78
|
constructor(lmdb) {
|
|
63
79
|
this.lmdb = lmdb;
|
|
64
|
-
|
|
65
|
-
// $FlowFixMe
|
|
66
|
-
this[Symbol.dispose] = () => {
|
|
67
|
-
this.lmdb.close();
|
|
68
|
-
};
|
|
69
80
|
}
|
|
70
81
|
has(key) {
|
|
71
82
|
return this.lmdb.hasSync(key);
|
|
@@ -80,23 +91,40 @@ class LmdbWrapper {
|
|
|
80
91
|
const buffer = typeof value === 'string' ? Buffer.from(value) : value;
|
|
81
92
|
await this.lmdb.put(key, buffer);
|
|
82
93
|
}
|
|
83
|
-
|
|
94
|
+
*keys() {
|
|
95
|
+
const PAGE_SIZE = 10000000;
|
|
96
|
+
let currentKeys = this.lmdb.keysSync(0, PAGE_SIZE);
|
|
97
|
+
while (currentKeys.length > 0) {
|
|
98
|
+
for (const key of currentKeys) {
|
|
99
|
+
yield key;
|
|
100
|
+
}
|
|
101
|
+
currentKeys = this.lmdb.keysSync(currentKeys.length, PAGE_SIZE);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
compact(targetPath) {
|
|
105
|
+
this.lmdb.compact(targetPath);
|
|
106
|
+
}
|
|
84
107
|
}
|
|
85
108
|
exports.LmdbWrapper = LmdbWrapper;
|
|
86
|
-
function open(directory
|
|
109
|
+
function open(directory,
|
|
87
110
|
// eslint-disable-next-line no-unused-vars
|
|
88
|
-
) {
|
|
111
|
+
openOptions) {
|
|
89
112
|
return new LmdbWrapper(new (_rust().Lmdb)({
|
|
90
113
|
path: directory,
|
|
91
114
|
asyncWrites: true,
|
|
92
|
-
mapSize: 1024 * 1024 * 1024 * 15
|
|
115
|
+
mapSize: process.env.ATLASPACK_BUILD_ENV === 'test' ? 1024 * 1024 * 1024 : 1024 * 1024 * 1024 * 15
|
|
93
116
|
}));
|
|
94
117
|
}
|
|
95
118
|
const pipeline = (0, _util().promisify)(_stream().default.pipeline);
|
|
96
119
|
class LMDBLiteCache {
|
|
120
|
+
/**
|
|
121
|
+
* Directory where we store raw files.
|
|
122
|
+
*/
|
|
123
|
+
|
|
97
124
|
constructor(cacheDir) {
|
|
98
125
|
this.fs = new (_fs().NodeFS)();
|
|
99
126
|
this.dir = cacheDir;
|
|
127
|
+
this.cacheFilesDirectory = _path().default.join(cacheDir, 'files');
|
|
100
128
|
this.fsCache = new _FSCache.FSCache(this.fs, cacheDir);
|
|
101
129
|
this.store = open(cacheDir, {
|
|
102
130
|
name: 'parcel-cache',
|
|
@@ -115,6 +143,7 @@ class LMDBLiteCache {
|
|
|
115
143
|
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
|
116
144
|
await this.fsCache.ensure();
|
|
117
145
|
}
|
|
146
|
+
await this.fs.mkdirp(this.cacheFilesDirectory);
|
|
118
147
|
return Promise.resolve();
|
|
119
148
|
}
|
|
120
149
|
serialize() {
|
|
@@ -139,10 +168,18 @@ class LMDBLiteCache {
|
|
|
139
168
|
await this.setBlob(key, (0, _buildCache().serialize)(value));
|
|
140
169
|
}
|
|
141
170
|
getStream(key) {
|
|
142
|
-
|
|
171
|
+
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
|
172
|
+
return this.fs.createReadStream(_path().default.join(this.dir, key));
|
|
173
|
+
}
|
|
174
|
+
return this.fs.createReadStream(this.getFileKey(key));
|
|
143
175
|
}
|
|
144
|
-
setStream(key, stream) {
|
|
145
|
-
|
|
176
|
+
async setStream(key, stream) {
|
|
177
|
+
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
|
178
|
+
return pipeline(stream, this.fs.createWriteStream(_path().default.join(this.dir, key)));
|
|
179
|
+
}
|
|
180
|
+
const filePath = this.getFileKey(key);
|
|
181
|
+
await this.fs.mkdirp(_path().default.dirname(filePath));
|
|
182
|
+
return pipeline(stream, this.fs.createWriteStream(filePath));
|
|
146
183
|
}
|
|
147
184
|
|
|
148
185
|
// eslint-disable-next-line require-await
|
|
@@ -162,34 +199,25 @@ class LMDBLiteCache {
|
|
|
162
199
|
getBuffer(key) {
|
|
163
200
|
return Promise.resolve(this.store.get(key));
|
|
164
201
|
}
|
|
165
|
-
#getFilePath(key, index) {
|
|
166
|
-
return _path().default.join(this.dir, `${key}-${index}`);
|
|
167
|
-
}
|
|
168
202
|
hasLargeBlob(key) {
|
|
169
203
|
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
|
170
204
|
return this.fsCache.hasLargeBlob(key);
|
|
171
205
|
}
|
|
172
|
-
return this.
|
|
206
|
+
return this.fs.exists(this.getFileKey(key));
|
|
173
207
|
}
|
|
174
|
-
|
|
175
|
-
/**
|
|
176
|
-
* @deprecated Use getBlob instead.
|
|
177
|
-
*/
|
|
178
208
|
getLargeBlob(key) {
|
|
179
209
|
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
|
180
210
|
return this.fsCache.getLargeBlob(key);
|
|
181
211
|
}
|
|
182
|
-
return
|
|
212
|
+
return this.fs.readFile(this.getFileKey(key));
|
|
183
213
|
}
|
|
184
|
-
|
|
185
|
-
/**
|
|
186
|
-
* @deprecated Use setBlob instead.
|
|
187
|
-
*/
|
|
188
|
-
setLargeBlob(key, contents, options) {
|
|
214
|
+
async setLargeBlob(key, contents, options) {
|
|
189
215
|
if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
|
|
190
216
|
return this.fsCache.setLargeBlob(key, contents, options);
|
|
191
217
|
}
|
|
192
|
-
|
|
218
|
+
const targetPath = this.getFileKey(key);
|
|
219
|
+
await this.fs.mkdirp(_path().default.dirname(targetPath));
|
|
220
|
+
return this.fs.writeFile(targetPath, contents);
|
|
193
221
|
}
|
|
194
222
|
|
|
195
223
|
/**
|
|
@@ -201,12 +229,53 @@ class LMDBLiteCache {
|
|
|
201
229
|
}
|
|
202
230
|
return this.store.delete(key);
|
|
203
231
|
}
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
this.
|
|
232
|
+
keys() {
|
|
233
|
+
return this.store.keys();
|
|
234
|
+
}
|
|
235
|
+
async compact(targetPath) {
|
|
236
|
+
await this.fs.mkdirp(targetPath);
|
|
237
|
+
const files = await this.fs.readdir(this.dir);
|
|
238
|
+
// copy all files except data.mdb and lock.mdb to the target path (recursive)
|
|
239
|
+
for (const file of files) {
|
|
240
|
+
const filePath = _path().default.join(this.dir, file);
|
|
241
|
+
if (file === 'data.mdb' || file === 'lock.mdb') {
|
|
242
|
+
continue;
|
|
243
|
+
}
|
|
244
|
+
await ncpAsync(filePath, _path().default.join(targetPath, file));
|
|
245
|
+
}
|
|
246
|
+
this.store.compact(_path().default.join(targetPath, 'data.mdb'));
|
|
247
|
+
}
|
|
248
|
+
refresh() {}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* Streams, packages are stored in files instead of LMDB.
|
|
252
|
+
*
|
|
253
|
+
* On this case, if a cache key happens to have a parent traversal, ../..
|
|
254
|
+
* it is treated specially
|
|
255
|
+
*
|
|
256
|
+
* That is, something/../something and something are meant to be different
|
|
257
|
+
* keys.
|
|
258
|
+
*
|
|
259
|
+
* Plus we do not want to store values outside of the cache directory.
|
|
260
|
+
*/
|
|
261
|
+
getFileKey(key) {
|
|
262
|
+
const cleanKey = key.split('/').map(part => {
|
|
263
|
+
if (part === '..') {
|
|
264
|
+
return '$$__parent_dir$$';
|
|
265
|
+
}
|
|
266
|
+
return part;
|
|
267
|
+
}).join('/');
|
|
268
|
+
return _path().default.join(this.cacheFilesDirectory, cleanKey);
|
|
269
|
+
}
|
|
270
|
+
async clear() {
|
|
271
|
+
await (0, _logger().instrumentAsync)('LMDBLiteCache::clear', async () => {
|
|
272
|
+
const keys = await this.keys();
|
|
273
|
+
for (const key of keys) {
|
|
274
|
+
await this.store.delete(key);
|
|
275
|
+
}
|
|
276
|
+
await this.fs.rimraf(this.cacheFilesDirectory);
|
|
277
|
+
await this.fs.mkdirp(this.cacheFilesDirectory);
|
|
278
|
+
});
|
|
210
279
|
}
|
|
211
280
|
}
|
|
212
281
|
exports.LMDBLiteCache = LMDBLiteCache;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { Readable } from 'stream';
|
|
2
|
+
import type { FilePath } from '@atlaspack/types';
|
|
3
|
+
import type { FileSystem } from '@atlaspack/fs';
|
|
4
|
+
import type { Cache } from './types';
|
|
5
|
+
export declare class FSCache implements Cache {
|
|
6
|
+
#private;
|
|
7
|
+
fs: FileSystem;
|
|
8
|
+
dir: FilePath;
|
|
9
|
+
constructor(fs: FileSystem, cacheDir: FilePath);
|
|
10
|
+
ensure(): Promise<void>;
|
|
11
|
+
_getCachePath(cacheId: string): FilePath;
|
|
12
|
+
getStream(key: string): Readable;
|
|
13
|
+
setStream(key: string, stream: Readable): Promise<void>;
|
|
14
|
+
has(key: string): Promise<boolean>;
|
|
15
|
+
getBlob(key: string): Promise<Buffer>;
|
|
16
|
+
setBlob(key: string, contents: Buffer | string): Promise<void>;
|
|
17
|
+
getBuffer(key: string): Promise<Buffer | null | undefined>;
|
|
18
|
+
hasLargeBlob(key: string): Promise<boolean>;
|
|
19
|
+
getLargeBlob(key: string): Promise<Buffer>;
|
|
20
|
+
setLargeBlob(key: string, contents: Buffer | string, options?: {
|
|
21
|
+
signal?: AbortSignal;
|
|
22
|
+
}): Promise<void>;
|
|
23
|
+
deleteLargeBlob(key: string): Promise<void>;
|
|
24
|
+
get<T>(key: string): Promise<T | null | undefined>;
|
|
25
|
+
set(key: string, value: unknown): Promise<void>;
|
|
26
|
+
refresh(): void;
|
|
27
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { Cache } from './types';
|
|
2
|
+
import { Readable } from 'stream';
|
|
3
|
+
export declare class IDBCache implements Cache {
|
|
4
|
+
store: any;
|
|
5
|
+
constructor();
|
|
6
|
+
ensure(): Promise<void>;
|
|
7
|
+
serialize(): Record<any, any>;
|
|
8
|
+
static deserialize(): IDBCache;
|
|
9
|
+
has(key: string): Promise<boolean>;
|
|
10
|
+
get<T>(key: string): Promise<T | null | undefined>;
|
|
11
|
+
set(key: string, value: unknown): Promise<void>;
|
|
12
|
+
getStream(key: string): Readable;
|
|
13
|
+
setStream(key: string, stream: Readable): Promise<void>;
|
|
14
|
+
getBlob(key: string): Promise<Buffer>;
|
|
15
|
+
setBlob(key: string, contents: Buffer | string): Promise<void>;
|
|
16
|
+
getBuffer(key: string): Promise<Buffer | null | undefined>;
|
|
17
|
+
hasLargeBlob(key: string): Promise<boolean>;
|
|
18
|
+
getLargeBlob(key: string): Promise<Buffer>;
|
|
19
|
+
setLargeBlob(key: string, contents: Buffer | string): Promise<void>;
|
|
20
|
+
deleteLargeBlob(key: string): Promise<void>;
|
|
21
|
+
refresh(): void;
|
|
22
|
+
}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { Lmdb } from '@atlaspack/rust';
|
|
2
|
+
import type { FilePath } from '@atlaspack/types';
|
|
3
|
+
import type { Cache } from './types';
|
|
4
|
+
import type { Readable } from 'stream';
|
|
5
|
+
import { NodeFS } from '@atlaspack/fs';
|
|
6
|
+
import { FSCache } from './FSCache';
|
|
7
|
+
interface DBOpenOptions {
|
|
8
|
+
name: string;
|
|
9
|
+
encoding: string;
|
|
10
|
+
compression: boolean;
|
|
11
|
+
}
|
|
12
|
+
export declare class LmdbWrapper {
|
|
13
|
+
lmdb: Lmdb;
|
|
14
|
+
constructor(lmdb: Lmdb);
|
|
15
|
+
has(key: string): boolean;
|
|
16
|
+
delete(key: string): Promise<void>;
|
|
17
|
+
get(key: string): Buffer | null;
|
|
18
|
+
put(key: string, value: Buffer | string): Promise<void>;
|
|
19
|
+
keys(): Iterable<string>;
|
|
20
|
+
compact(targetPath: string): void;
|
|
21
|
+
}
|
|
22
|
+
export declare function open(directory: string, openOptions: DBOpenOptions): LmdbWrapper;
|
|
23
|
+
export type SerLMDBLiteCache = {
|
|
24
|
+
dir: FilePath;
|
|
25
|
+
};
|
|
26
|
+
export declare class LMDBLiteCache implements Cache {
|
|
27
|
+
fs: NodeFS;
|
|
28
|
+
dir: FilePath;
|
|
29
|
+
store: LmdbWrapper;
|
|
30
|
+
fsCache: FSCache;
|
|
31
|
+
/**
|
|
32
|
+
* Directory where we store raw files.
|
|
33
|
+
*/
|
|
34
|
+
cacheFilesDirectory: FilePath;
|
|
35
|
+
constructor(cacheDir: FilePath);
|
|
36
|
+
/**
|
|
37
|
+
* Use this to pass the native LMDB instance back to Rust.
|
|
38
|
+
*/
|
|
39
|
+
getNativeRef(): Lmdb;
|
|
40
|
+
ensure(): Promise<void>;
|
|
41
|
+
serialize(): SerLMDBLiteCache;
|
|
42
|
+
static deserialize(cache: SerLMDBLiteCache): LMDBLiteCache;
|
|
43
|
+
has(key: string): Promise<boolean>;
|
|
44
|
+
get<T>(key: string): Promise<T | null | undefined>;
|
|
45
|
+
set(key: string, value: unknown): Promise<void>;
|
|
46
|
+
getStream(key: string): Readable;
|
|
47
|
+
setStream(key: string, stream: Readable): Promise<void>;
|
|
48
|
+
getBlob(key: string): Promise<Buffer>;
|
|
49
|
+
getBlobSync(key: string): Buffer;
|
|
50
|
+
setBlob(key: string, contents: Buffer | string): Promise<void>;
|
|
51
|
+
getBuffer(key: string): Promise<Buffer | null | undefined>;
|
|
52
|
+
hasLargeBlob(key: string): Promise<boolean>;
|
|
53
|
+
getLargeBlob(key: string): Promise<Buffer>;
|
|
54
|
+
setLargeBlob(key: string, contents: Buffer | string, options?: {
|
|
55
|
+
signal?: AbortSignal;
|
|
56
|
+
}): Promise<void>;
|
|
57
|
+
/**
|
|
58
|
+
* @deprecated Use store.delete instead.
|
|
59
|
+
*/
|
|
60
|
+
deleteLargeBlob(key: string): Promise<void>;
|
|
61
|
+
keys(): Iterable<string>;
|
|
62
|
+
compact(targetPath: string): Promise<void>;
|
|
63
|
+
refresh(): void;
|
|
64
|
+
/**
|
|
65
|
+
* Streams, packages are stored in files instead of LMDB.
|
|
66
|
+
*
|
|
67
|
+
* On this case, if a cache key happens to have a parent traversal, ../..
|
|
68
|
+
* it is treated specially
|
|
69
|
+
*
|
|
70
|
+
* That is, something/../something and something are meant to be different
|
|
71
|
+
* keys.
|
|
72
|
+
*
|
|
73
|
+
* Plus we do not want to store values outside of the cache directory.
|
|
74
|
+
*/
|
|
75
|
+
getFileKey(key: string): string;
|
|
76
|
+
clear(): Promise<void>;
|
|
77
|
+
}
|
|
78
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const WRITE_LIMIT_CHUNK: number;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@atlaspack/cache",
|
|
3
3
|
"description": "Interface for defining caches and file-system, IDB and LMDB implementations.",
|
|
4
|
-
"version": "3.1.1-canary.
|
|
4
|
+
"version": "3.1.1-canary.300+2f3af9374",
|
|
5
5
|
"license": "(MIT OR Apache-2.0)",
|
|
6
6
|
"type": "commonjs",
|
|
7
7
|
"publishConfig": {
|
|
@@ -11,24 +11,24 @@
|
|
|
11
11
|
"type": "git",
|
|
12
12
|
"url": "https://github.com/atlassian-labs/atlaspack.git"
|
|
13
13
|
},
|
|
14
|
-
"main": "lib/index.js",
|
|
15
|
-
"source": "src/index.
|
|
16
|
-
"types": "index.d.ts",
|
|
14
|
+
"main": "./lib/index.js",
|
|
15
|
+
"source": "./src/index.ts",
|
|
16
|
+
"types": "./lib/types/index.d.ts",
|
|
17
17
|
"engines": {
|
|
18
18
|
"node": ">= 16.0.0"
|
|
19
19
|
},
|
|
20
20
|
"scripts": {
|
|
21
21
|
"test": "mocha",
|
|
22
|
-
"build
|
|
23
|
-
"check-ts": "tsc --noEmit index.d.ts"
|
|
22
|
+
"build:lib": "gulp build --gulpfile ../../../gulpfile.js --cwd ."
|
|
24
23
|
},
|
|
25
24
|
"dependencies": {
|
|
26
|
-
"@atlaspack/build-cache": "2.13.3-canary.
|
|
27
|
-
"@atlaspack/feature-flags": "2.14.1-canary.
|
|
28
|
-
"@atlaspack/fs": "2.14.5-canary.
|
|
29
|
-
"@atlaspack/logger": "2.14.5-canary.
|
|
30
|
-
"@atlaspack/rust": "3.2.1-canary.
|
|
31
|
-
"@atlaspack/utils": "2.14.5-canary.
|
|
25
|
+
"@atlaspack/build-cache": "2.13.3-canary.368+2f3af9374",
|
|
26
|
+
"@atlaspack/feature-flags": "2.14.1-canary.368+2f3af9374",
|
|
27
|
+
"@atlaspack/fs": "2.14.5-canary.300+2f3af9374",
|
|
28
|
+
"@atlaspack/logger": "2.14.5-canary.300+2f3af9374",
|
|
29
|
+
"@atlaspack/rust": "3.2.1-canary.300+2f3af9374",
|
|
30
|
+
"@atlaspack/utils": "2.14.5-canary.300+2f3af9374",
|
|
31
|
+
"ncp": "^2.0.0"
|
|
32
32
|
},
|
|
33
33
|
"devDependencies": {
|
|
34
34
|
"idb": "^5.0.8"
|
|
@@ -36,5 +36,5 @@
|
|
|
36
36
|
"browser": {
|
|
37
37
|
"./src/IDBCache.js": "./src/IDBCache.browser.js"
|
|
38
38
|
},
|
|
39
|
-
"gitHead": "
|
|
40
|
-
}
|
|
39
|
+
"gitHead": "2f3af9374bcc082e6f4835f6f5a2b98e32ea45b4"
|
|
40
|
+
}
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
// @flow strict-local
|
|
2
|
-
|
|
3
1
|
import type {Readable, Writable} from 'stream';
|
|
4
2
|
import type {FilePath} from '@atlaspack/types';
|
|
5
3
|
import type {FileSystem} from '@atlaspack/fs';
|
|
@@ -9,6 +7,8 @@ import stream from 'stream';
|
|
|
9
7
|
import path from 'path';
|
|
10
8
|
import {promisify} from 'util';
|
|
11
9
|
|
|
10
|
+
import {hashString} from '@atlaspack/rust';
|
|
11
|
+
import {getFeatureFlag} from '@atlaspack/feature-flags';
|
|
12
12
|
import logger from '@atlaspack/logger';
|
|
13
13
|
import {
|
|
14
14
|
deserialize,
|
|
@@ -21,7 +21,7 @@ import packageJson from '../package.json';
|
|
|
21
21
|
|
|
22
22
|
import {WRITE_LIMIT_CHUNK} from './constants';
|
|
23
23
|
|
|
24
|
-
const pipeline: (Readable, Writable) => Promise<void> = promisify(
|
|
24
|
+
const pipeline: (arg1: Readable, arg2: Writable) => Promise<void> = promisify(
|
|
25
25
|
stream.pipeline,
|
|
26
26
|
);
|
|
27
27
|
|
|
@@ -40,9 +40,10 @@ export class FSCache implements Cache {
|
|
|
40
40
|
|
|
41
41
|
// In parallel, create sub-directories for every possible hex value
|
|
42
42
|
// This speeds up large caches on many file systems since there are fewer files in a single directory.
|
|
43
|
-
let dirPromises = [];
|
|
43
|
+
let dirPromises: Array<Promise<undefined>> = [];
|
|
44
44
|
for (let i = 0; i < 256; i++) {
|
|
45
45
|
dirPromises.push(
|
|
46
|
+
// @ts-expect-error TS2345
|
|
46
47
|
this.fs.mkdirp(path.join(this.dir, ('00' + i.toString(16)).slice(-2))),
|
|
47
48
|
);
|
|
48
49
|
}
|
|
@@ -51,6 +52,10 @@ export class FSCache implements Cache {
|
|
|
51
52
|
}
|
|
52
53
|
|
|
53
54
|
_getCachePath(cacheId: string): FilePath {
|
|
55
|
+
if (getFeatureFlag('cachePerformanceImprovements')) {
|
|
56
|
+
const cleanId = hashString(cacheId);
|
|
57
|
+
return path.join(this.dir, cleanId.slice(0, 2), cleanId.slice(2));
|
|
58
|
+
}
|
|
54
59
|
return path.join(this.dir, cacheId.slice(0, 2), cacheId.slice(2));
|
|
55
60
|
}
|
|
56
61
|
|
|
@@ -77,10 +82,10 @@ export class FSCache implements Cache {
|
|
|
77
82
|
await this.fs.writeFile(this._getCachePath(key), contents);
|
|
78
83
|
}
|
|
79
84
|
|
|
80
|
-
async getBuffer(key: string): Promise
|
|
85
|
+
async getBuffer(key: string): Promise<Buffer | null | undefined> {
|
|
81
86
|
try {
|
|
82
87
|
return await this.fs.readFile(this._getCachePath(key));
|
|
83
|
-
} catch (err) {
|
|
88
|
+
} catch (err: any) {
|
|
84
89
|
if (err.code === 'ENOENT') {
|
|
85
90
|
return null;
|
|
86
91
|
} else {
|
|
@@ -90,14 +95,17 @@ export class FSCache implements Cache {
|
|
|
90
95
|
}
|
|
91
96
|
|
|
92
97
|
#getFilePath(key: string, index: number): string {
|
|
98
|
+
if (getFeatureFlag('cachePerformanceImprovements')) {
|
|
99
|
+
return path.join(this.dir, `${hashString(key)}-${index}`);
|
|
100
|
+
}
|
|
93
101
|
return path.join(this.dir, `${key}-${index}`);
|
|
94
102
|
}
|
|
95
103
|
|
|
96
|
-
async #unlinkChunks(key: string, index: number): Promise<
|
|
104
|
+
async #unlinkChunks(key: string, index: number): Promise<undefined> {
|
|
97
105
|
try {
|
|
98
106
|
await this.fs.unlink(this.#getFilePath(key, index));
|
|
99
107
|
await this.#unlinkChunks(key, index + 1);
|
|
100
|
-
} catch (err) {
|
|
108
|
+
} catch (err: any) {
|
|
101
109
|
// If there's an error, no more chunks are left to delete
|
|
102
110
|
}
|
|
103
111
|
}
|
|
@@ -120,21 +128,26 @@ export class FSCache implements Cache {
|
|
|
120
128
|
async setLargeBlob(
|
|
121
129
|
key: string,
|
|
122
130
|
contents: Buffer | string,
|
|
123
|
-
options?: {
|
|
131
|
+
options?: {
|
|
132
|
+
signal?: AbortSignal;
|
|
133
|
+
},
|
|
124
134
|
): Promise<void> {
|
|
125
135
|
const chunks = Math.ceil(contents.length / WRITE_LIMIT_CHUNK);
|
|
126
136
|
|
|
127
|
-
const writePromises: Promise<
|
|
137
|
+
const writePromises: Promise<undefined>[] = [];
|
|
128
138
|
if (chunks === 1) {
|
|
129
139
|
// If there's one chunk, don't slice the content
|
|
130
140
|
writePromises.push(
|
|
141
|
+
// @ts-expect-error TS2345
|
|
131
142
|
this.fs.writeFile(this.#getFilePath(key, 0), contents, {
|
|
143
|
+
// @ts-expect-error TS2353
|
|
132
144
|
signal: options?.signal,
|
|
133
145
|
}),
|
|
134
146
|
);
|
|
135
147
|
} else {
|
|
136
148
|
for (let i = 0; i < chunks; i += 1) {
|
|
137
149
|
writePromises.push(
|
|
150
|
+
// @ts-expect-error TS2345
|
|
138
151
|
this.fs.writeFile(
|
|
139
152
|
this.#getFilePath(key, i),
|
|
140
153
|
typeof contents === 'string'
|
|
@@ -146,6 +159,7 @@ export class FSCache implements Cache {
|
|
|
146
159
|
i * WRITE_LIMIT_CHUNK,
|
|
147
160
|
(i + 1) * WRITE_LIMIT_CHUNK,
|
|
148
161
|
),
|
|
162
|
+
// @ts-expect-error TS2353
|
|
149
163
|
{signal: options?.signal},
|
|
150
164
|
),
|
|
151
165
|
);
|
|
@@ -159,12 +173,13 @@ export class FSCache implements Cache {
|
|
|
159
173
|
}
|
|
160
174
|
|
|
161
175
|
async deleteLargeBlob(key: string): Promise<void> {
|
|
162
|
-
const deletePromises: Promise<
|
|
176
|
+
const deletePromises: Promise<undefined>[] = [];
|
|
163
177
|
|
|
164
178
|
let i = 0;
|
|
165
179
|
let filePath = this.#getFilePath(key, i);
|
|
166
180
|
|
|
167
181
|
while (await this.fs.exists(filePath)) {
|
|
182
|
+
// @ts-expect-error TS2345
|
|
168
183
|
deletePromises.push(this.fs.rimraf(filePath));
|
|
169
184
|
i += 1;
|
|
170
185
|
filePath = this.#getFilePath(key, i);
|
|
@@ -173,11 +188,11 @@ export class FSCache implements Cache {
|
|
|
173
188
|
await Promise.all(deletePromises);
|
|
174
189
|
}
|
|
175
190
|
|
|
176
|
-
async get<T>(key: string): Promise
|
|
191
|
+
async get<T>(key: string): Promise<T | null | undefined> {
|
|
177
192
|
try {
|
|
178
193
|
let data = await this.fs.readFile(this._getCachePath(key));
|
|
179
194
|
return deserialize(data);
|
|
180
|
-
} catch (err) {
|
|
195
|
+
} catch (err: any) {
|
|
181
196
|
if (err.code === 'ENOENT') {
|
|
182
197
|
return null;
|
|
183
198
|
} else {
|
|
@@ -186,13 +201,13 @@ export class FSCache implements Cache {
|
|
|
186
201
|
}
|
|
187
202
|
}
|
|
188
203
|
|
|
189
|
-
async set(key: string, value:
|
|
204
|
+
async set(key: string, value: unknown): Promise<void> {
|
|
190
205
|
try {
|
|
191
206
|
let blobPath = this._getCachePath(key);
|
|
192
207
|
let data = serialize(value);
|
|
193
208
|
|
|
194
209
|
await this.fs.writeFile(blobPath, data);
|
|
195
|
-
} catch (err) {
|
|
210
|
+
} catch (err: any) {
|
|
196
211
|
logger.error(err, '@atlaspack/cache');
|
|
197
212
|
}
|
|
198
213
|
}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
// @flow strict-local
|
|
2
1
|
import type {Cache} from './types';
|
|
3
2
|
|
|
4
3
|
import {Readable} from 'stream';
|
|
@@ -9,21 +8,18 @@ import {
|
|
|
9
8
|
serialize,
|
|
10
9
|
} from '@atlaspack/build-cache';
|
|
11
10
|
import {bufferStream} from '@atlaspack/utils';
|
|
12
|
-
// $FlowFixMe[untyped-import]
|
|
13
11
|
import {openDB} from 'idb';
|
|
14
12
|
|
|
15
|
-
// $FlowFixMe[untyped-import]
|
|
16
13
|
import packageJson from '../package.json';
|
|
17
14
|
|
|
18
15
|
const STORE_NAME = 'cache';
|
|
19
16
|
|
|
20
17
|
export class IDBCache implements Cache {
|
|
21
|
-
// $FlowFixMe
|
|
22
18
|
store: any;
|
|
23
19
|
|
|
24
20
|
constructor() {
|
|
25
21
|
this.store = openDB('REPL-parcel-cache', 1, {
|
|
26
|
-
upgrade(db) {
|
|
22
|
+
upgrade(db: any) {
|
|
27
23
|
db.createObjectStore(STORE_NAME);
|
|
28
24
|
},
|
|
29
25
|
blocked() {},
|
|
@@ -36,7 +32,7 @@ export class IDBCache implements Cache {
|
|
|
36
32
|
return Promise.resolve();
|
|
37
33
|
}
|
|
38
34
|
|
|
39
|
-
serialize():
|
|
35
|
+
serialize(): Record<any, any> {
|
|
40
36
|
return {
|
|
41
37
|
/*::...null*/
|
|
42
38
|
};
|
|
@@ -50,7 +46,7 @@ export class IDBCache implements Cache {
|
|
|
50
46
|
return Promise.resolve(this.store.get(key) != null);
|
|
51
47
|
}
|
|
52
48
|
|
|
53
|
-
async get<T>(key: string): Promise
|
|
49
|
+
async get<T>(key: string): Promise<T | null | undefined> {
|
|
54
50
|
let data = await (await this.store).get(STORE_NAME, key);
|
|
55
51
|
if (data == null) {
|
|
56
52
|
return null;
|
|
@@ -59,17 +55,19 @@ export class IDBCache implements Cache {
|
|
|
59
55
|
return Promise.resolve(deserialize(data));
|
|
60
56
|
}
|
|
61
57
|
|
|
62
|
-
async set(key: string, value:
|
|
58
|
+
async set(key: string, value: unknown): Promise<void> {
|
|
63
59
|
await (await this.store).put(STORE_NAME, serialize(value), key);
|
|
64
60
|
}
|
|
65
61
|
|
|
66
62
|
getStream(key: string): Readable {
|
|
67
63
|
let dataPromise = this.store
|
|
64
|
+
// @ts-expect-error TS7006
|
|
68
65
|
.then((s) => s.get(STORE_NAME, key))
|
|
66
|
+
// @ts-expect-error TS7006
|
|
69
67
|
.then((d) => Buffer.from(d))
|
|
68
|
+
// @ts-expect-error TS7006
|
|
70
69
|
.catch((e) => e);
|
|
71
70
|
const stream = new Readable({
|
|
72
|
-
// $FlowFixMe(incompatible-call)
|
|
73
71
|
async read() {
|
|
74
72
|
let data = await dataPromise;
|
|
75
73
|
if (data instanceof Error) {
|
|
@@ -118,7 +116,7 @@ export class IDBCache implements Cache {
|
|
|
118
116
|
// ]);
|
|
119
117
|
// }
|
|
120
118
|
|
|
121
|
-
async getBuffer(key: string): Promise
|
|
119
|
+
async getBuffer(key: string): Promise<Buffer | null | undefined> {
|
|
122
120
|
let data = await (await this.store).get(STORE_NAME, key);
|
|
123
121
|
if (data == null) {
|
|
124
122
|
return null;
|