@atlaspack/cache 3.1.1-canary.48 → 3.1.1-canary.481
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +602 -0
- package/dist/FSCache.js +162 -0
- package/dist/IDBCache.browser.js +124 -0
- package/dist/IDBCache.js +10 -0
- package/dist/LMDBLiteCache.js +198 -0
- package/dist/constants.js +5 -0
- package/dist/index.js +19 -0
- package/dist/types.js +2 -0
- package/lib/FSCache.js +15 -25
- package/lib/IDBCache.browser.js +10 -7
- package/lib/IDBCache.js +1 -1
- package/lib/LMDBLiteCache.js +20 -63
- package/lib/types/FSCache.d.ts +27 -0
- package/lib/types/IDBCache.browser.d.ts +22 -0
- package/lib/types/IDBCache.d.ts +4 -0
- package/lib/types/LMDBLiteCache.d.ts +78 -0
- package/lib/types/constants.d.ts +1 -0
- package/lib/types/index.d.ts +4 -0
- package/lib/types/types.d.ts +2 -0
- package/package.json +14 -15
- package/src/{FSCache.js → FSCache.ts} +21 -24
- package/src/{IDBCache.browser.js → IDBCache.browser.ts} +9 -10
- package/src/{IDBCache.js → IDBCache.ts} +1 -2
- package/src/{LMDBLiteCache.js → LMDBLiteCache.ts} +26 -60
- package/src/{constants.js → constants.ts} +0 -2
- package/src/{index.js → index.ts} +0 -2
- package/src/{types.js → types.ts} +0 -1
- package/test/{LMDBLiteCache.test.js → LMDBLiteCache.test.ts} +9 -11
- package/tsconfig.json +27 -0
- package/tsconfig.tsbuildinfo +1 -0
- package/index.d.ts +0 -12
- package/lib/types.d.ts +0 -2
|
@@ -1,22 +1,18 @@
|
|
|
1
|
-
// @flow strict-local
|
|
2
|
-
|
|
3
1
|
import {
|
|
4
2
|
deserialize,
|
|
5
3
|
registerSerializableClass,
|
|
6
4
|
serialize,
|
|
7
5
|
} from '@atlaspack/build-cache';
|
|
8
|
-
import {getFeatureFlag} from '@atlaspack/feature-flags';
|
|
9
6
|
import {Lmdb} from '@atlaspack/rust';
|
|
10
7
|
import type {FilePath} from '@atlaspack/types';
|
|
11
8
|
import type {Cache} from './types';
|
|
12
9
|
import type {Readable, Writable} from 'stream';
|
|
13
|
-
|
|
10
|
+
// @ts-expect-error TS7016
|
|
14
11
|
import ncp from 'ncp';
|
|
15
12
|
import {promisify} from 'util';
|
|
16
13
|
import stream from 'stream';
|
|
17
14
|
import path from 'path';
|
|
18
15
|
import {NodeFS} from '@atlaspack/fs';
|
|
19
|
-
// $FlowFixMe
|
|
20
16
|
import packageJson from '../package.json';
|
|
21
17
|
import {FSCache} from './FSCache';
|
|
22
18
|
import {instrumentAsync} from '@atlaspack/logger';
|
|
@@ -84,19 +80,19 @@ export function open(
|
|
|
84
80
|
asyncWrites: true,
|
|
85
81
|
mapSize:
|
|
86
82
|
process.env.ATLASPACK_BUILD_ENV === 'test'
|
|
87
|
-
?
|
|
83
|
+
? 256 * 1024 * 1024
|
|
88
84
|
: 1024 * 1024 * 1024 * 15,
|
|
89
85
|
}),
|
|
90
86
|
);
|
|
91
87
|
}
|
|
92
88
|
|
|
93
|
-
const pipeline: (Readable, Writable) => Promise<void> = promisify(
|
|
89
|
+
const pipeline: (arg1: Readable, arg2: Writable) => Promise<void> = promisify(
|
|
94
90
|
stream.pipeline,
|
|
95
91
|
);
|
|
96
92
|
|
|
97
|
-
export type SerLMDBLiteCache = {
|
|
98
|
-
dir: FilePath
|
|
99
|
-
|
|
93
|
+
export type SerLMDBLiteCache = {
|
|
94
|
+
dir: FilePath;
|
|
95
|
+
};
|
|
100
96
|
|
|
101
97
|
export class LMDBLiteCache implements Cache {
|
|
102
98
|
fs: NodeFS;
|
|
@@ -129,9 +125,7 @@ export class LMDBLiteCache implements Cache {
|
|
|
129
125
|
}
|
|
130
126
|
|
|
131
127
|
async ensure(): Promise<void> {
|
|
132
|
-
|
|
133
|
-
await this.fsCache.ensure();
|
|
134
|
-
}
|
|
128
|
+
await this.fsCache.ensure();
|
|
135
129
|
await this.fs.mkdirp(this.cacheFilesDirectory);
|
|
136
130
|
return Promise.resolve();
|
|
137
131
|
}
|
|
@@ -150,7 +144,7 @@ export class LMDBLiteCache implements Cache {
|
|
|
150
144
|
return Promise.resolve(this.store.has(key));
|
|
151
145
|
}
|
|
152
146
|
|
|
153
|
-
get<T>(key: string): Promise
|
|
147
|
+
get<T>(key: string): Promise<T | null | undefined> {
|
|
154
148
|
let data = this.store.get(key);
|
|
155
149
|
if (data == null) {
|
|
156
150
|
return Promise.resolve(null);
|
|
@@ -159,29 +153,19 @@ export class LMDBLiteCache implements Cache {
|
|
|
159
153
|
return Promise.resolve(deserialize(data));
|
|
160
154
|
}
|
|
161
155
|
|
|
162
|
-
async set(key: string, value:
|
|
156
|
+
async set(key: string, value: unknown): Promise<void> {
|
|
163
157
|
await this.setBlob(key, serialize(value));
|
|
164
158
|
}
|
|
165
159
|
|
|
166
160
|
getStream(key: string): Readable {
|
|
167
|
-
|
|
168
|
-
return this.fs.createReadStream(path.join(this.dir, key));
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
return fs.createReadStream(this.getFileKey(key));
|
|
161
|
+
return this.fs.createReadStream(path.join(this.dir, key));
|
|
172
162
|
}
|
|
173
163
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
);
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
const filePath = this.getFileKey(key);
|
|
183
|
-
await fs.promises.mkdir(path.dirname(filePath), {recursive: true});
|
|
184
|
-
return pipeline(stream, fs.createWriteStream(filePath));
|
|
164
|
+
setStream(key: string, stream: Readable): Promise<void> {
|
|
165
|
+
return pipeline(
|
|
166
|
+
stream,
|
|
167
|
+
this.fs.createWriteStream(path.join(this.dir, key)),
|
|
168
|
+
);
|
|
185
169
|
}
|
|
186
170
|
|
|
187
171
|
// eslint-disable-next-line require-await
|
|
@@ -201,51 +185,33 @@ export class LMDBLiteCache implements Cache {
|
|
|
201
185
|
await this.store.put(key, contents);
|
|
202
186
|
}
|
|
203
187
|
|
|
204
|
-
getBuffer(key: string): Promise
|
|
188
|
+
getBuffer(key: string): Promise<Buffer | null | undefined> {
|
|
205
189
|
return Promise.resolve(this.store.get(key));
|
|
206
190
|
}
|
|
207
191
|
|
|
208
192
|
hasLargeBlob(key: string): Promise<boolean> {
|
|
209
|
-
|
|
210
|
-
return this.fsCache.hasLargeBlob(key);
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
return fs.promises
|
|
214
|
-
.access(this.getFileKey(key), fs.constants.F_OK)
|
|
215
|
-
.then(() => true)
|
|
216
|
-
.catch(() => false);
|
|
193
|
+
return this.fsCache.hasLargeBlob(key);
|
|
217
194
|
}
|
|
218
195
|
|
|
219
196
|
getLargeBlob(key: string): Promise<Buffer> {
|
|
220
|
-
|
|
221
|
-
return this.fsCache.getLargeBlob(key);
|
|
222
|
-
}
|
|
223
|
-
return fs.promises.readFile(this.getFileKey(key));
|
|
197
|
+
return this.fsCache.getLargeBlob(key);
|
|
224
198
|
}
|
|
225
199
|
|
|
226
|
-
|
|
200
|
+
setLargeBlob(
|
|
227
201
|
key: string,
|
|
228
202
|
contents: Buffer | string,
|
|
229
|
-
options?: {
|
|
203
|
+
options?: {
|
|
204
|
+
signal?: AbortSignal;
|
|
205
|
+
},
|
|
230
206
|
): Promise<void> {
|
|
231
|
-
|
|
232
|
-
return this.fsCache.setLargeBlob(key, contents, options);
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
const targetPath = this.getFileKey(key);
|
|
236
|
-
await fs.promises.mkdir(path.dirname(targetPath), {recursive: true});
|
|
237
|
-
return fs.promises.writeFile(targetPath, contents);
|
|
207
|
+
return this.fsCache.setLargeBlob(key, contents, options);
|
|
238
208
|
}
|
|
239
209
|
|
|
240
210
|
/**
|
|
241
211
|
* @deprecated Use store.delete instead.
|
|
242
212
|
*/
|
|
243
213
|
deleteLargeBlob(key: string): Promise<void> {
|
|
244
|
-
|
|
245
|
-
return this.fsCache.deleteLargeBlob(key);
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
return this.store.delete(key);
|
|
214
|
+
return this.fsCache.deleteLargeBlob(key);
|
|
249
215
|
}
|
|
250
216
|
|
|
251
217
|
keys(): Iterable<string> {
|
|
@@ -253,9 +219,9 @@ export class LMDBLiteCache implements Cache {
|
|
|
253
219
|
}
|
|
254
220
|
|
|
255
221
|
async compact(targetPath: string): Promise<void> {
|
|
256
|
-
await fs.
|
|
222
|
+
await this.fs.mkdirp(targetPath);
|
|
257
223
|
|
|
258
|
-
const files = await fs.
|
|
224
|
+
const files = await this.fs.readdir(this.dir);
|
|
259
225
|
// copy all files except data.mdb and lock.mdb to the target path (recursive)
|
|
260
226
|
for (const file of files) {
|
|
261
227
|
const filePath = path.join(this.dir, file);
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
// @flow
|
|
2
|
-
|
|
3
1
|
import * as fs from 'fs';
|
|
4
2
|
import * as path from 'path';
|
|
5
3
|
import {tmpdir} from 'os';
|
|
@@ -12,7 +10,7 @@ import {initializeMonitoring} from '@atlaspack/rust';
|
|
|
12
10
|
const cacheDir = path.join(tmpdir(), 'lmdb-lite-cache-tests');
|
|
13
11
|
|
|
14
12
|
describe('LMDBLiteCache', () => {
|
|
15
|
-
let cache;
|
|
13
|
+
let cache: any;
|
|
16
14
|
|
|
17
15
|
beforeEach(async () => {
|
|
18
16
|
await fs.promises.rm(cacheDir, {recursive: true, force: true});
|
|
@@ -129,7 +127,7 @@ describe('LMDBLiteCache', () => {
|
|
|
129
127
|
await cache.ensure();
|
|
130
128
|
await cache.setBlob(`key${i}`, Buffer.from(serialize({value: i})));
|
|
131
129
|
|
|
132
|
-
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
130
|
+
await new Promise((resolve: any) => setTimeout(resolve, 10));
|
|
133
131
|
}
|
|
134
132
|
|
|
135
133
|
const finalCache = new LMDBLiteCache(testDir);
|
|
@@ -144,7 +142,7 @@ describe('LMDBLiteCache', () => {
|
|
|
144
142
|
|
|
145
143
|
try {
|
|
146
144
|
initializeMonitoring();
|
|
147
|
-
} catch (error) {
|
|
145
|
+
} catch (error: any) {
|
|
148
146
|
/* empty */
|
|
149
147
|
}
|
|
150
148
|
|
|
@@ -165,8 +163,8 @@ describe('LMDBLiteCache', () => {
|
|
|
165
163
|
|
|
166
164
|
const numWorkers = 10;
|
|
167
165
|
|
|
168
|
-
const workers = [];
|
|
169
|
-
const responsePromises = [];
|
|
166
|
+
const workers: Array<any> = [];
|
|
167
|
+
const responsePromises: Array<any> = [];
|
|
170
168
|
for (let i = 0; i < numWorkers; i++) {
|
|
171
169
|
const worker = new Worker(path.join(__dirname, 'workerThreadsTest.js'), {
|
|
172
170
|
workerData: {
|
|
@@ -175,16 +173,16 @@ describe('LMDBLiteCache', () => {
|
|
|
175
173
|
});
|
|
176
174
|
workers.push(worker);
|
|
177
175
|
|
|
178
|
-
const responsePromise = new Promise((resolve, reject) => {
|
|
176
|
+
const responsePromise = new Promise((resolve: any, reject: any) => {
|
|
179
177
|
worker.addListener('error', (error: Error) => {
|
|
180
178
|
reject(error);
|
|
181
179
|
});
|
|
182
|
-
worker.addListener('message', (message) => {
|
|
180
|
+
worker.addListener('message', (message: any) => {
|
|
183
181
|
resolve(message);
|
|
184
182
|
});
|
|
185
183
|
});
|
|
186
184
|
|
|
187
|
-
worker.addListener('message', (message) => {
|
|
185
|
+
worker.addListener('message', (message: any) => {
|
|
188
186
|
// eslint-disable-next-line no-console
|
|
189
187
|
console.log('Worker message', message);
|
|
190
188
|
});
|
|
@@ -236,7 +234,7 @@ describe('LMDBLiteCache', () => {
|
|
|
236
234
|
workerId: worker.threadId,
|
|
237
235
|
});
|
|
238
236
|
|
|
239
|
-
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
237
|
+
await new Promise((resolve: any) => setTimeout(resolve, 500));
|
|
240
238
|
worker.terminate();
|
|
241
239
|
}
|
|
242
240
|
});
|
package/tsconfig.json
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"extends": "../../../tsconfig.base.json",
|
|
3
|
+
"include": ["./src/", "./package.json"],
|
|
4
|
+
"compilerOptions": {
|
|
5
|
+
"composite": true
|
|
6
|
+
},
|
|
7
|
+
"references": [
|
|
8
|
+
{
|
|
9
|
+
"path": "../build-cache/tsconfig.json"
|
|
10
|
+
},
|
|
11
|
+
{
|
|
12
|
+
"path": "../feature-flags/tsconfig.json"
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"path": "../fs/tsconfig.json"
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
"path": "../logger/tsconfig.json"
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"path": "../rust/tsconfig.json"
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"path": "../utils/tsconfig.json"
|
|
25
|
+
}
|
|
26
|
+
]
|
|
27
|
+
}
|