@bjoernboss/mws 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.txt +29 -0
- package/README.md +301 -0
- package/dist/base.d.ts +198 -0
- package/dist/base.js +73 -0
- package/dist/builder.d.ts +58 -0
- package/dist/builder.js +146 -0
- package/dist/cache.d.ts +66 -0
- package/dist/cache.js +708 -0
- package/dist/client.d.ts +228 -0
- package/dist/client.js +1646 -0
- package/dist/handler.d.ts +119 -0
- package/dist/handler.js +542 -0
- package/dist/helper.d.ts +33 -0
- package/dist/helper.js +363 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +12 -0
- package/dist/log.d.ts +52 -0
- package/dist/log.js +288 -0
- package/dist/server.d.ts +66 -0
- package/dist/server.js +257 -0
- package/package.json +46 -0
package/dist/cache.js
ADDED
|
@@ -0,0 +1,708 @@
|
|
|
1
|
+
/* SPDX-License-Identifier: BSD-3-Clause */
|
|
2
|
+
/* Copyright (c) 2026 Bjoern Boss Henrichsen */
|
|
3
|
+
import * as libLog from "./log.js";
|
|
4
|
+
import * as libHelper from "./helper.js";
|
|
5
|
+
import * as libFs from "fs";
|
|
6
|
+
import * as libFsPromises from "fs/promises";
|
|
7
|
+
import * as libStream from "stream";
|
|
8
|
+
import * as libCrypto from "crypto";
|
|
9
|
+
const UNIQUE_ID_CHARS = '0123456789abcdefghijklmnopqrstuvwxyz';
|
|
10
|
+
const UNIQUE_ID_LENGTH = 14;
|
|
11
|
+
const ID_EXTENSION_REGEX = RegExp(`^\\.[${UNIQUE_ID_CHARS}]{${UNIQUE_ID_LENGTH}}$`, 'i');
|
|
12
|
+
function readStats(path) {
|
|
13
|
+
try {
|
|
14
|
+
const stats = libFs.statSync(path);
|
|
15
|
+
if (stats.isFile())
|
|
16
|
+
return [stats.size, stats.mtimeMs];
|
|
17
|
+
}
|
|
18
|
+
catch (err) {
|
|
19
|
+
if (err.code == 'ENOENT')
|
|
20
|
+
return null;
|
|
21
|
+
throw new Error(`Filesystem error while checking [${path}]: ${err.message}`);
|
|
22
|
+
}
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
25
|
+
async function atomicWrite(path, content, logger, options) {
|
|
26
|
+
const tempPath = (options?.temporary ?? `${path}.temp`);
|
|
27
|
+
let written = false;
|
|
28
|
+
try {
|
|
29
|
+
logger.trace(`Writing ${options?.what ?? 'data'} to [${path}]`);
|
|
30
|
+
/* write the content to the temporary file */
|
|
31
|
+
await libFsPromises.writeFile(tempPath, content);
|
|
32
|
+
written = true;
|
|
33
|
+
await libFsPromises.rename(tempPath, path);
|
|
34
|
+
return true;
|
|
35
|
+
}
|
|
36
|
+
catch (err) {
|
|
37
|
+
if (written)
|
|
38
|
+
logger.error(`Failed to replace the original file [${path}]: ${err.message}`);
|
|
39
|
+
else
|
|
40
|
+
logger.error(`Failed to write to temporary file [${tempPath}]: ${err.message}`);
|
|
41
|
+
try {
|
|
42
|
+
await libFsPromises.unlink(tempPath);
|
|
43
|
+
}
|
|
44
|
+
catch (err) {
|
|
45
|
+
if (err.code != 'ENOENT')
|
|
46
|
+
logger.warning(`Failed to remove temporary file [${tempPath}]: ${err.message}`);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return false;
|
|
50
|
+
}
|
|
51
|
+
class CacheManager {
|
|
52
|
+
logger;
|
|
53
|
+
map = {};
|
|
54
|
+
nextStamp = 0;
|
|
55
|
+
allocated = 0;
|
|
56
|
+
totalCapacity;
|
|
57
|
+
largestSize;
|
|
58
|
+
nextAge;
|
|
59
|
+
constructor(logger, cacheSize, fileSizeLimit) {
|
|
60
|
+
this.logger = logger;
|
|
61
|
+
this.totalCapacity = cacheSize;
|
|
62
|
+
this.largestSize = fileSizeLimit;
|
|
63
|
+
this.nextAge = 0;
|
|
64
|
+
}
|
|
65
|
+
reduce(maximum) {
|
|
66
|
+
/* create the list of all cached objects sorted by the touched count */
|
|
67
|
+
const paths = Object.keys(this.map).sort((a, b) => this.map[a].touched - this.map[b].touched);
|
|
68
|
+
/* drop the first half of the entries or more, if the size has not yet been freed */
|
|
69
|
+
for (let i = 0; i < (paths.length + 1) / 2 || this.allocated > maximum; ++i)
|
|
70
|
+
this.drop(paths[i]);
|
|
71
|
+
}
|
|
72
|
+
add(path, data, mtime, age) {
|
|
73
|
+
if (!this.cacheable(data.byteLength))
|
|
74
|
+
return;
|
|
75
|
+
/* check if the entry is already part of the cache and check if the current entry should be evicted */
|
|
76
|
+
if (path in this.map) {
|
|
77
|
+
const entry = this.map[path];
|
|
78
|
+
if (entry.age >= age)
|
|
79
|
+
return;
|
|
80
|
+
this.drop(path);
|
|
81
|
+
}
|
|
82
|
+
/* check if space needs to be reserved */
|
|
83
|
+
if (this.allocated + data.byteLength > this.totalCapacity)
|
|
84
|
+
this.reduce(this.totalCapacity - data.byteLength);
|
|
85
|
+
/* add the entry to the cache */
|
|
86
|
+
this.logger.log(`Added [${path}] to the cache (Size: ${data.byteLength})`);
|
|
87
|
+
this.allocated += data.byteLength;
|
|
88
|
+
this.map[path] = { data, encodings: {}, mtime, touched: ++this.nextStamp, age };
|
|
89
|
+
}
|
|
90
|
+
addEncoding(path, data, age, name) {
|
|
91
|
+
if (!this.cacheable(data.byteLength))
|
|
92
|
+
return;
|
|
93
|
+
/* check if the entry is still in the cache */
|
|
94
|
+
if (!(path in this.map) || this.map[path].age != age)
|
|
95
|
+
return;
|
|
96
|
+
/* check if the encoding already exists */
|
|
97
|
+
if (name in this.map[path].encodings)
|
|
98
|
+
return;
|
|
99
|
+
/* check if space needs to be reserved and check if the root entry is still available afterwards */
|
|
100
|
+
if (this.allocated + data.byteLength > this.totalCapacity)
|
|
101
|
+
this.reduce(this.totalCapacity - data.byteLength);
|
|
102
|
+
if (!(path in this.map))
|
|
103
|
+
return;
|
|
104
|
+
/* add the entry to the cache */
|
|
105
|
+
this.logger.log(`Added encoding [${name}] of [${path}] to the cache (Size: ${data.byteLength})`);
|
|
106
|
+
this.allocated += data.byteLength;
|
|
107
|
+
this.map[path].encodings[name] = data;
|
|
108
|
+
}
|
|
109
|
+
drop(path) {
|
|
110
|
+
if (!(path in this.map))
|
|
111
|
+
return;
|
|
112
|
+
this.logger.log(`Dropped [${path}] and encodings from the cache`);
|
|
113
|
+
/* remove all cached encodings and the entry itself */
|
|
114
|
+
const entry = this.map[path];
|
|
115
|
+
for (const key in entry.encodings)
|
|
116
|
+
this.allocated -= entry.encodings[key].byteLength;
|
|
117
|
+
this.allocated -= entry.data.byteLength;
|
|
118
|
+
delete this.map[path];
|
|
119
|
+
}
|
|
120
|
+
cacheable(size) {
|
|
121
|
+
return (size <= this.largestSize && size <= this.totalCapacity);
|
|
122
|
+
}
|
|
123
|
+
flush() {
|
|
124
|
+
this.map = {};
|
|
125
|
+
this.allocated = 0;
|
|
126
|
+
}
|
|
127
|
+
find(path, stats) {
|
|
128
|
+
const entry = this.map[path] ?? null;
|
|
129
|
+
if (entry == null)
|
|
130
|
+
return null;
|
|
131
|
+
/* check if the entry is still up-to-date and return it */
|
|
132
|
+
if (stats == null || (entry.mtime == stats.mtime && entry.data.length == stats.size)) {
|
|
133
|
+
entry.touched = ++this.nextStamp;
|
|
134
|
+
return entry;
|
|
135
|
+
}
|
|
136
|
+
/* remove the entry as it seems to be outdated */
|
|
137
|
+
this.drop(path);
|
|
138
|
+
return null;
|
|
139
|
+
}
|
|
140
|
+
allocAge() {
|
|
141
|
+
return ++this.nextAge;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
class ImmutableManager {
|
|
145
|
+
logger;
|
|
146
|
+
map = {};
|
|
147
|
+
reverse = {};
|
|
148
|
+
writeBack = null;
|
|
149
|
+
alwaysValidate;
|
|
150
|
+
immutableTagging;
|
|
151
|
+
constructor(writeBackPath, logger, alwaysValidate, immutableTagging) {
|
|
152
|
+
this.logger = logger;
|
|
153
|
+
this.alwaysValidate = alwaysValidate;
|
|
154
|
+
this.immutableTagging = immutableTagging;
|
|
155
|
+
if (writeBackPath != '')
|
|
156
|
+
this.configureWriteBack(writeBackPath);
|
|
157
|
+
}
|
|
158
|
+
assignId(entry) {
|
|
159
|
+
const firstId = (entry.unique == '');
|
|
160
|
+
if (!firstId)
|
|
161
|
+
delete this.reverse[entry.unique];
|
|
162
|
+
/* setup the new unique id */
|
|
163
|
+
entry.unique = '';
|
|
164
|
+
for (let i = 0; i < UNIQUE_ID_LENGTH; ++i)
|
|
165
|
+
entry.unique += UNIQUE_ID_CHARS[libCrypto.randomInt(UNIQUE_ID_CHARS.length)];
|
|
166
|
+
/* patch the state up to contain the new id */
|
|
167
|
+
const [base, name, extension] = libHelper.splitFilePath(entry.path);
|
|
168
|
+
entry.immutable = `${base}${name}.${entry.unique}${extension}`;
|
|
169
|
+
this.reverse[entry.unique] = entry.identifier;
|
|
170
|
+
this.logger.trace(`${firstId ? 'Allocated' : 'Re-allocated'} immutable unique id [${entry.unique}] for [${entry.identifier}]`);
|
|
171
|
+
}
|
|
172
|
+
async storeState() {
|
|
173
|
+
if (this.writeBack == null)
|
|
174
|
+
return;
|
|
175
|
+
this.writeBack.dirty = true;
|
|
176
|
+
if (this.writeBack.writing != null)
|
|
177
|
+
return;
|
|
178
|
+
let resolver = () => { };
|
|
179
|
+
this.writeBack.writing = new Promise((res) => resolver = res);
|
|
180
|
+
/* check if the state is dirty and perform the write backs */
|
|
181
|
+
while (this.writeBack.dirty) {
|
|
182
|
+
this.writeBack.dirty = false;
|
|
183
|
+
/* collect the list of all relevant entries */
|
|
184
|
+
let output = [];
|
|
185
|
+
for (const identifier in this.map) {
|
|
186
|
+
const entry = this.map[identifier];
|
|
187
|
+
output.push({
|
|
188
|
+
unique: entry.unique,
|
|
189
|
+
identifier,
|
|
190
|
+
path: entry.path,
|
|
191
|
+
fileSystem: entry.fileSystem,
|
|
192
|
+
size: entry.size,
|
|
193
|
+
mtime: entry.mtime
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
const content = JSON.stringify(output);
|
|
197
|
+
/* ignore any read/write failures */
|
|
198
|
+
await atomicWrite(this.writeBack.path, Buffer.from(content, 'utf-8'), this.logger, { what: 'immutable state' });
|
|
199
|
+
}
|
|
200
|
+
this.writeBack.writing = null;
|
|
201
|
+
resolver();
|
|
202
|
+
}
|
|
203
|
+
async loadState(path) {
|
|
204
|
+
this.logger.trace(`Loading immutable state from [${path}]`);
|
|
205
|
+
/* load the current file and parse it as json (skip any content failed to be read or if the file did not exist) */
|
|
206
|
+
let state = null;
|
|
207
|
+
try {
|
|
208
|
+
state = JSON.parse(await libFsPromises.readFile(path, { encoding: 'utf-8' }));
|
|
209
|
+
}
|
|
210
|
+
catch (err) {
|
|
211
|
+
if (err.code != 'ENOENT')
|
|
212
|
+
this.logger.error(`Error while loading immutable state from [${path}]: ${err.message}`);
|
|
213
|
+
return [];
|
|
214
|
+
}
|
|
215
|
+
if (!Array.isArray(state)) {
|
|
216
|
+
this.logger.error(`Immutable state [${path}] is malformed (discarding state)`);
|
|
217
|
+
return [];
|
|
218
|
+
}
|
|
219
|
+
/* parse all of the values and validate their general structure */
|
|
220
|
+
let corrupted = 0, output = [];
|
|
221
|
+
for (const entry of state) {
|
|
222
|
+
if (typeof entry.unique != 'string' || typeof entry.identifier != 'string' || (typeof entry.fileSystem != 'string' && entry.fileSystem !== null))
|
|
223
|
+
++corrupted;
|
|
224
|
+
else if (!`.${entry.unique}`.match(ID_EXTENSION_REGEX) || typeof entry.path != 'string')
|
|
225
|
+
++corrupted;
|
|
226
|
+
else if (typeof entry.mtime != 'number' || !isFinite(entry.mtime) || entry.mtime < 0)
|
|
227
|
+
++corrupted;
|
|
228
|
+
else if (typeof entry.size != 'number' || !isFinite(entry.size) || entry.size < 0 || Math.floor(entry.size) != entry.size)
|
|
229
|
+
++corrupted;
|
|
230
|
+
else
|
|
231
|
+
output.push({ unique: entry.unique, identifier: entry.identifier, path: entry.path, fileSystem: entry.fileSystem, size: entry.size, mtime: entry.mtime });
|
|
232
|
+
}
|
|
233
|
+
if (corrupted > 0)
|
|
234
|
+
this.logger.error(`Immutable loaded state [${path}] contained [${corrupted}] malformed entires`);
|
|
235
|
+
return output;
|
|
236
|
+
}
|
|
237
|
+
updateEntry(entry, checkFreshness, firstAssign) {
|
|
238
|
+
if (entry.fetched && !checkFreshness && !this.alwaysValidate)
|
|
239
|
+
return true;
|
|
240
|
+
/* fetch the file states of the actual filesystem entry */
|
|
241
|
+
const stats = readStats(entry.fileSystem);
|
|
242
|
+
if (stats == null) {
|
|
243
|
+
this.logger.warning(`Immutable path [${entry.fileSystem}] does not exist`);
|
|
244
|
+
delete this.map[entry.identifier];
|
|
245
|
+
delete this.reverse[entry.unique];
|
|
246
|
+
this.storeState();
|
|
247
|
+
return false;
|
|
248
|
+
}
|
|
249
|
+
const [fileSize, mtime] = stats;
|
|
250
|
+
entry.fetched = true;
|
|
251
|
+
/* check if the stats have changed, in which case a new unique-id needs to be assigned (thus binding the it to the stats) */
|
|
252
|
+
if (!firstAssign) {
|
|
253
|
+
if (entry.size == fileSize && entry.mtime == mtime)
|
|
254
|
+
return true;
|
|
255
|
+
this.assignId(entry);
|
|
256
|
+
}
|
|
257
|
+
entry.size = fileSize;
|
|
258
|
+
entry.mtime = mtime;
|
|
259
|
+
this.storeState();
|
|
260
|
+
return true;
|
|
261
|
+
}
|
|
262
|
+
async configureWriteBack(path) {
|
|
263
|
+
let states = await this.loadState(path);
|
|
264
|
+
/* merge the loaded state into the current state (prefer current state over
|
|
265
|
+
* loaded state; current state might already exist after awaiting the load) */
|
|
266
|
+
const performInitialWriteBack = (Object.keys(this.map).length > 0);
|
|
267
|
+
for (const state of states) {
|
|
268
|
+
if (state.identifier in this.map)
|
|
269
|
+
continue;
|
|
270
|
+
this.logger.trace(`Recovering immutable unique id [${state.unique}] for [${state.identifier}]`);
|
|
271
|
+
const [base, name, extension] = libHelper.splitFilePath(state.path);
|
|
272
|
+
this.map[state.identifier] = {
|
|
273
|
+
immutable: `${base}${name}.${state.unique}${extension}`,
|
|
274
|
+
identifier: state.identifier,
|
|
275
|
+
path: state.path,
|
|
276
|
+
fileSystem: state.fileSystem,
|
|
277
|
+
size: state.size,
|
|
278
|
+
mtime: state.mtime,
|
|
279
|
+
unique: state.unique,
|
|
280
|
+
fetched: false
|
|
281
|
+
};
|
|
282
|
+
this.reverse[state.unique] = state.identifier;
|
|
283
|
+
}
|
|
284
|
+
/* configure the actual writeback and check if an initial store needs to be triggered */
|
|
285
|
+
this.writeBack = { path, writing: null, dirty: false };
|
|
286
|
+
if (performInitialWriteBack)
|
|
287
|
+
this.storeState();
|
|
288
|
+
}
|
|
289
|
+
make(handler, path, checkFreshness) {
|
|
290
|
+
const identifier = `${handler}:${path}`;
|
|
291
|
+
if (!this.immutableTagging)
|
|
292
|
+
return path;
|
|
293
|
+
while (true) {
|
|
294
|
+
/* check if the entry does not yet exist and the id-tagged path needs to be created */
|
|
295
|
+
let entry = this.map[identifier] ?? null;
|
|
296
|
+
if (entry == null) {
|
|
297
|
+
entry = (this.map[identifier] = { immutable: '', identifier, path, unique: '', fileSystem: null, size: 0, mtime: 0, fetched: false });
|
|
298
|
+
this.assignId(entry);
|
|
299
|
+
this.storeState();
|
|
300
|
+
}
|
|
301
|
+
/* check if the stats can actually be validated or if the entry should just be served (if
|
|
302
|
+
* the file does not exist/has been removed, simply restart the loop with to get a fresh id) */
|
|
303
|
+
else if (entry.fileSystem != null) {
|
|
304
|
+
try {
|
|
305
|
+
if (!this.updateEntry(entry, checkFreshness, false))
|
|
306
|
+
continue;
|
|
307
|
+
}
|
|
308
|
+
catch (err) {
|
|
309
|
+
this.logger.warning(`Failed to validate immutable entry [${identifier}] and assuming unmodified: ${err.message}`);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
return entry.immutable;
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
get(path, checkFreshness) {
|
|
316
|
+
if (!this.immutableTagging)
|
|
317
|
+
return [null, false];
|
|
318
|
+
/* check if it might be an immutable-tagged path */
|
|
319
|
+
const [base, temp, extension] = libHelper.splitFilePath(path);
|
|
320
|
+
const [_, name, tempId] = libHelper.splitFilePath(temp);
|
|
321
|
+
if (!tempId.match(ID_EXTENSION_REGEX))
|
|
322
|
+
return [null, false];
|
|
323
|
+
const unique = tempId.substring(1);
|
|
324
|
+
/* check if the entry does not exist in the reverse mapping anymore - indicating that the unique-id
|
|
325
|
+
* is old/outdated, in which case it can be recovered by comparing the full actual path to all of
|
|
326
|
+
* the entries, looking for the actual object, thereby recovering the most recent immutable path */
|
|
327
|
+
let identifier = this.reverse[unique] ?? null;
|
|
328
|
+
if (identifier == null) {
|
|
329
|
+
const fileSystemPath = `${base}${name}${extension}`;
|
|
330
|
+
for (const tempIdentifier in this.map) {
|
|
331
|
+
if (this.map[tempIdentifier].fileSystem != fileSystemPath)
|
|
332
|
+
continue;
|
|
333
|
+
identifier = tempIdentifier;
|
|
334
|
+
break;
|
|
335
|
+
}
|
|
336
|
+
if (identifier == null)
|
|
337
|
+
return [null, false];
|
|
338
|
+
}
|
|
339
|
+
const entry = this.map[identifier], firstAssign = (entry.fileSystem == null);
|
|
340
|
+
if (entry.fileSystem == null)
|
|
341
|
+
entry.fileSystem = `${base}${name}${extension}`;
|
|
342
|
+
/* update the entry and return the final stats/if the unique-id changed */
|
|
343
|
+
if (!this.updateEntry(entry, checkFreshness, firstAssign))
|
|
344
|
+
return [null, false];
|
|
345
|
+
return [entry, entry.unique != unique];
|
|
346
|
+
}
|
|
347
|
+
invalidate() {
|
|
348
|
+
for (const identifier in this.map)
|
|
349
|
+
this.map[identifier].fetched = false;
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
class AlreadyCached {
|
|
353
|
+
cache;
|
|
354
|
+
path;
|
|
355
|
+
entry;
|
|
356
|
+
immutable;
|
|
357
|
+
constructor(cache, path, entry, immutable) {
|
|
358
|
+
this.cache = cache;
|
|
359
|
+
this.path = path;
|
|
360
|
+
this.entry = entry;
|
|
361
|
+
this.immutable = immutable;
|
|
362
|
+
}
|
|
363
|
+
isImmutable() {
|
|
364
|
+
return this.immutable;
|
|
365
|
+
}
|
|
366
|
+
filePath() {
|
|
367
|
+
return this.path;
|
|
368
|
+
}
|
|
369
|
+
fileSize() {
|
|
370
|
+
return this.entry.data.byteLength;
|
|
371
|
+
}
|
|
372
|
+
lastModified() {
|
|
373
|
+
return new Date(this.entry.mtime).toUTCString();
|
|
374
|
+
}
|
|
375
|
+
uniqueId() {
|
|
376
|
+
return `${this.entry.mtime}-${this.entry.data.byteLength}`;
|
|
377
|
+
}
|
|
378
|
+
stream(options) {
|
|
379
|
+
return libStream.Readable.from(this.entry.data.subarray(options?.start, (options?.end == null ? undefined : options.end + 1)));
|
|
380
|
+
}
|
|
381
|
+
async read() {
|
|
382
|
+
return this.entry.data;
|
|
383
|
+
}
|
|
384
|
+
readSync() {
|
|
385
|
+
return this.entry.data;
|
|
386
|
+
}
|
|
387
|
+
encoded(encoding) {
|
|
388
|
+
return EncodedCache(this.cache, this, this.entry, this.entry.age, encoding);
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
class NotCached {
|
|
392
|
+
cache;
|
|
393
|
+
path;
|
|
394
|
+
size;
|
|
395
|
+
mtime;
|
|
396
|
+
immutable;
|
|
397
|
+
age;
|
|
398
|
+
constructor(cache, path, size, mtime, age, immutable) {
|
|
399
|
+
this.cache = cache;
|
|
400
|
+
this.path = path;
|
|
401
|
+
this.size = size;
|
|
402
|
+
this.mtime = mtime;
|
|
403
|
+
this.immutable = immutable;
|
|
404
|
+
this.age = age;
|
|
405
|
+
}
|
|
406
|
+
makeStream(options) {
|
|
407
|
+
/* create the data stream */
|
|
408
|
+
let stream = null;
|
|
409
|
+
try {
|
|
410
|
+
stream = libFs.createReadStream(this.path, { flags: 'r', start: options?.start, end: options?.end });
|
|
411
|
+
}
|
|
412
|
+
catch (err) {
|
|
413
|
+
return new libStream.Readable({ read() { this.destroy(new Error(`Error reading file: ${err.message}`)); } });
|
|
414
|
+
}
|
|
415
|
+
/* check if only a partial file is being read, or it is too large, in which case it will not be added to the cache */
|
|
416
|
+
if (!this.cache.cacheable(this.size) || (options?.start != null && options.start != 0) || (options?.end != null && options.end + 1 != this.size))
|
|
417
|
+
return stream;
|
|
418
|
+
/* create the transformer stream to cache the data */
|
|
419
|
+
let buffers = [], settled = false, totalLength = 0;
|
|
420
|
+
const sniffer = new libStream.Transform({
|
|
421
|
+
transform: (chunk, _, cb) => {
|
|
422
|
+
if (settled)
|
|
423
|
+
return cb(new Error('Reading already completed'));
|
|
424
|
+
totalLength += chunk.byteLength;
|
|
425
|
+
buffers.push(chunk);
|
|
426
|
+
cb(null, chunk);
|
|
427
|
+
},
|
|
428
|
+
final: (cb) => {
|
|
429
|
+
if (settled)
|
|
430
|
+
return cb(new Error('Reading already completed'));
|
|
431
|
+
if (totalLength == this.size)
|
|
432
|
+
this.cache.add(this.path, Buffer.concat(buffers), this.mtime, this.age);
|
|
433
|
+
cb(null);
|
|
434
|
+
}
|
|
435
|
+
});
|
|
436
|
+
/* setup the file exceptions to be propagated to the stream */
|
|
437
|
+
stream.pipe(sniffer);
|
|
438
|
+
stream.once('error', (err) => {
|
|
439
|
+
if (settled)
|
|
440
|
+
return;
|
|
441
|
+
settled = true;
|
|
442
|
+
sniffer.destroy(err);
|
|
443
|
+
});
|
|
444
|
+
sniffer.once('error', (err) => {
|
|
445
|
+
if (settled)
|
|
446
|
+
return;
|
|
447
|
+
settled = true;
|
|
448
|
+
stream.destroy(err);
|
|
449
|
+
});
|
|
450
|
+
return sniffer;
|
|
451
|
+
}
|
|
452
|
+
isImmutable() {
|
|
453
|
+
return this.immutable;
|
|
454
|
+
}
|
|
455
|
+
filePath() {
|
|
456
|
+
return this.path;
|
|
457
|
+
}
|
|
458
|
+
fileSize() {
|
|
459
|
+
return this.size;
|
|
460
|
+
}
|
|
461
|
+
lastModified() {
|
|
462
|
+
return new Date(this.mtime).toUTCString();
|
|
463
|
+
}
|
|
464
|
+
uniqueId() {
|
|
465
|
+
return `${this.mtime}-${this.size}`;
|
|
466
|
+
}
|
|
467
|
+
stream(options) {
|
|
468
|
+
return this.makeStream(options);
|
|
469
|
+
}
|
|
470
|
+
async read() {
|
|
471
|
+
return StreamToAsync(this.makeStream());
|
|
472
|
+
}
|
|
473
|
+
readSync() {
|
|
474
|
+
/* just let the errors propagate out */
|
|
475
|
+
const data = libFs.readFileSync(this.path);
|
|
476
|
+
/* add the read buffer back to the cache (using the fetched data from before reading the file - to
|
|
477
|
+
* detect a file-change since before reading the file; dont error as stream also just proceeds) */
|
|
478
|
+
if (data.byteLength == this.size)
|
|
479
|
+
this.cache.add(this.path, data, this.mtime, this.age);
|
|
480
|
+
return data;
|
|
481
|
+
}
|
|
482
|
+
encoded(encoding) {
|
|
483
|
+
return EncodedCache(this.cache, this, null, this.age, encoding);
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
function StreamToAsync(stream) {
|
|
487
|
+
return new Promise((resolve, reject) => {
|
|
488
|
+
const buffers = [];
|
|
489
|
+
let settled = false;
|
|
490
|
+
/* register the handler to collect the data and stream them out */
|
|
491
|
+
stream.on("data", (chunk) => {
|
|
492
|
+
if (!settled)
|
|
493
|
+
buffers.push(chunk);
|
|
494
|
+
});
|
|
495
|
+
stream.once("error", (err) => {
|
|
496
|
+
if (!settled) {
|
|
497
|
+
settled = true;
|
|
498
|
+
reject(err);
|
|
499
|
+
}
|
|
500
|
+
});
|
|
501
|
+
stream.once("end", () => {
|
|
502
|
+
if (!settled) {
|
|
503
|
+
settled = true;
|
|
504
|
+
resolve(Buffer.concat(buffers));
|
|
505
|
+
}
|
|
506
|
+
});
|
|
507
|
+
});
|
|
508
|
+
}
|
|
509
|
+
function EncodedCache(cache, reader, entry, age, encoding) {
|
|
510
|
+
/* check if no encoding is used, in which case the source reader can just be wrapped */
|
|
511
|
+
if (encoding == null) {
|
|
512
|
+
return {
|
|
513
|
+
contentSize: () => reader.fileSize(),
|
|
514
|
+
stream: () => reader.stream(),
|
|
515
|
+
read: () => reader.read(),
|
|
516
|
+
readSync: () => reader.readSync()
|
|
517
|
+
};
|
|
518
|
+
}
|
|
519
|
+
/* check if the given encoding has already been cached */
|
|
520
|
+
if (entry != null && encoding.name in entry.encodings) {
|
|
521
|
+
const encoded = entry.encodings[encoding.name];
|
|
522
|
+
return {
|
|
523
|
+
contentSize: () => encoded.byteLength,
|
|
524
|
+
stream: () => libStream.Readable.from(encoded),
|
|
525
|
+
read: async () => encoded,
|
|
526
|
+
readSync: () => encoded
|
|
527
|
+
};
|
|
528
|
+
}
|
|
529
|
+
const makeStream = () => {
|
|
530
|
+
let settled = false;
|
|
531
|
+
/* create the encoded pipe */
|
|
532
|
+
const stream = reader.stream();
|
|
533
|
+
const encoder = encoding.makeEncode();
|
|
534
|
+
let sniffer = null;
|
|
535
|
+
stream.pipe(encoder);
|
|
536
|
+
/* setup the stream exceptions to be propgated through */
|
|
537
|
+
stream.once('error', (err) => {
|
|
538
|
+
if (settled)
|
|
539
|
+
return;
|
|
540
|
+
settled = true;
|
|
541
|
+
encoder.destroy(err);
|
|
542
|
+
if (sniffer != null)
|
|
543
|
+
sniffer.destroy(err);
|
|
544
|
+
});
|
|
545
|
+
encoder.once('error', (err) => {
|
|
546
|
+
if (settled)
|
|
547
|
+
return;
|
|
548
|
+
settled = true;
|
|
549
|
+
stream.destroy(err);
|
|
550
|
+
if (sniffer != null)
|
|
551
|
+
sniffer.destroy(err);
|
|
552
|
+
});
|
|
553
|
+
/* check if there is even a chance for the data to be cached */
|
|
554
|
+
if (!cache.cacheable(reader.fileSize()))
|
|
555
|
+
return encoder;
|
|
556
|
+
/* setup the sniffer stream to collect the cached data */
|
|
557
|
+
let buffers = [];
|
|
558
|
+
sniffer = new libStream.Transform({
|
|
559
|
+
transform: (chunk, _, cb) => {
|
|
560
|
+
if (settled)
|
|
561
|
+
return cb(new Error('Reading already completed'));
|
|
562
|
+
buffers.push(chunk);
|
|
563
|
+
cb(null, chunk);
|
|
564
|
+
},
|
|
565
|
+
final: (cb) => {
|
|
566
|
+
if (settled)
|
|
567
|
+
return cb(new Error('Reading already completed'));
|
|
568
|
+
cache.addEncoding(reader.filePath(), Buffer.concat(buffers), age, encoding.name);
|
|
569
|
+
cb(null);
|
|
570
|
+
}
|
|
571
|
+
});
|
|
572
|
+
/* setup the stream exceptions to be propgated through */
|
|
573
|
+
sniffer.once('error', (err) => {
|
|
574
|
+
if (settled)
|
|
575
|
+
return;
|
|
576
|
+
settled = true;
|
|
577
|
+
stream.destroy(err);
|
|
578
|
+
encoder.destroy(err);
|
|
579
|
+
});
|
|
580
|
+
return encoder.pipe(sniffer);
|
|
581
|
+
};
|
|
582
|
+
/* wrap the original reader around the encoder (let errors propagate out) */
|
|
583
|
+
return {
|
|
584
|
+
contentSize: () => null,
|
|
585
|
+
stream: () => makeStream(),
|
|
586
|
+
read: () => StreamToAsync(makeStream()),
|
|
587
|
+
readSync: () => {
|
|
588
|
+
/* will be returned as buffer anyways, so might as well
|
|
589
|
+
* try to write it back to the cache, no matter if its too large */
|
|
590
|
+
const data = encoding.encodeBuffer(reader.readSync());
|
|
591
|
+
cache.addEncoding(reader.filePath(), data, age, encoding.name);
|
|
592
|
+
return data;
|
|
593
|
+
}
|
|
594
|
+
};
|
|
595
|
+
}
|
|
596
|
+
export class CacheHost extends libLog.Logger {
|
|
597
|
+
_cacheManager;
|
|
598
|
+
_immutableManager;
|
|
599
|
+
_config;
|
|
600
|
+
constructor(config) {
|
|
601
|
+
super('cache');
|
|
602
|
+
this.info('Cache created');
|
|
603
|
+
this._config = BurntCacheConfig.from(config);
|
|
604
|
+
this._cacheManager = new CacheManager(this, this._config.cacheSize, this._config.fileSizeLimit);
|
|
605
|
+
this._immutableManager = new ImmutableManager(this._config.immutableStatePath, this, this._config.alwaysValidate, this._config.immutableTagging);
|
|
606
|
+
}
|
|
607
|
+
resolveCache(path, checkFreshness, checkImmutable) {
|
|
608
|
+
/* check if the entry is immutable and fetch its actual path or
|
|
609
|
+
* if it has been moved (due to the id having been invalidated) */
|
|
610
|
+
const [immutable, immutableMoved] = (checkImmutable ? this._immutableManager.get(path, checkFreshness) : [null, false]);
|
|
611
|
+
if (immutable != null) {
|
|
612
|
+
path = immutable.fileSystem;
|
|
613
|
+
if (immutableMoved)
|
|
614
|
+
return immutable.immutable;
|
|
615
|
+
}
|
|
616
|
+
const isImmutable = (immutable != null);
|
|
617
|
+
/* check if does not need to be checked and the entry is already in the cache, in which case the file doesn't even need to be checked */
|
|
618
|
+
let entry = ((checkFreshness || this._config.alwaysValidate) ? null : this._cacheManager.find(path, null));
|
|
619
|
+
if (entry != null && (immutable == null || (immutable.size == entry.data.byteLength && immutable.mtime == entry.mtime)))
|
|
620
|
+
return new AlreadyCached(this._cacheManager, path, entry, isImmutable);
|
|
621
|
+
/* check if the file exists and read its file-size and mtime (not necessary
|
|
622
|
+
* for immutable entries, as their stats are already up-to-date) */
|
|
623
|
+
const stats = (immutable == null ? readStats(path) : [immutable.size, immutable.mtime]);
|
|
624
|
+
if (stats == null)
|
|
625
|
+
return null;
|
|
626
|
+
const [fileSize, mtime] = stats;
|
|
627
|
+
/* check if the path exists in the validated cache and otherwise return the uncached entry */
|
|
628
|
+
entry = this._cacheManager.find(path, { mtime, size: fileSize });
|
|
629
|
+
if (entry != null)
|
|
630
|
+
return new AlreadyCached(this._cacheManager, path, entry, isImmutable);
|
|
631
|
+
return new NotCached(this._cacheManager, path, fileSize, mtime, this._cacheManager.allocAge(), isImmutable);
|
|
632
|
+
}
|
|
633
|
+
/* configuration used by this cache host */
|
|
634
|
+
get config() {
|
|
635
|
+
return this._config;
|
|
636
|
+
}
|
|
637
|
+
/* [throws] if [checkFreshness] is true, re-validate the file stats on disk before serving from cache (defaults
|
|
638
|
+
* to false); resolve immutable ids automatically (Cached to interact with cache; null, if it does not exist,
|
|
639
|
+
* string if the immutable path has been permanently moved to the new path in source space) */
|
|
640
|
+
fetchImmutable(path, options) {
|
|
641
|
+
return this.resolveCache(path, options?.checkFreshness ?? false, true);
|
|
642
|
+
}
|
|
643
|
+
/* [throws] if [checkFreshness] is true re-validate the file stats on disk before serving from cache (defaults
|
|
644
|
+
* to false); no immutable ids are resolved (Cached to interact with cache; null, if it does not exist) */
|
|
645
|
+
fetchDirect(path, options) {
|
|
646
|
+
return this.resolveCache(path, options?.checkFreshness ?? false, false);
|
|
647
|
+
}
|
|
648
|
+
/* generate a unique tagged path for the given query path, which will change whenever the underlying file changes;
|
|
649
|
+
* [checkFreshness]: if true, re-validate the file stats on disk to detect changes (defaults to false); creates
|
|
650
|
+
* a path to a file, which looks similar to the source, except that the name includes a unique id, which will be used
|
|
651
|
+
* to identity the given file state (will be removed from the final target path to be served, to identify the actual source) */
|
|
652
|
+
immutable(handler, path, options) {
|
|
653
|
+
return this._immutableManager.make(handler, path, options?.checkFreshness ?? false);
|
|
654
|
+
}
|
|
655
|
+
/* flush all cached data and invalidate immutable stats so they are re-checked on next access */
|
|
656
|
+
flush() {
|
|
657
|
+
this.info('Flushing cache and invalidating immutable entries');
|
|
658
|
+
this._cacheManager.flush();
|
|
659
|
+
this._immutableManager.invalidate();
|
|
660
|
+
}
|
|
661
|
+
/* [throws] read the data directly into a buffer (designed for modules to interact with) */
|
|
662
|
+
async read(path, options) {
|
|
663
|
+
const entry = this.resolveCache(path, options?.checkFreshness ?? false, false);
|
|
664
|
+
if (entry == null)
|
|
665
|
+
return null;
|
|
666
|
+
return entry.read();
|
|
667
|
+
}
|
|
668
|
+
/* [throws] write data atomically to the disk and update the cache (designed for modules to interact
|
|
669
|
+
* with; writes as utf-8; writes data first to temporary file and then replaces the file atomically) */
|
|
670
|
+
async write(path, data, options) {
|
|
671
|
+
if (typeof data == 'string')
|
|
672
|
+
data = Buffer.from(data, 'utf-8');
|
|
673
|
+
/* write the data atomically to the destination */
|
|
674
|
+
if (!await atomicWrite(path, data, this, { what: (options?.what ?? 'via cache'), temporary: options?.temporary }))
|
|
675
|
+
throw new Error('Failed to atomically write data');
|
|
676
|
+
if (!this._cacheManager.cacheable(data.byteLength))
|
|
677
|
+
return;
|
|
678
|
+
const age = this._cacheManager.allocAge();
|
|
679
|
+
/* fetch the new state and update the cache (let errors propagate out; only if the write-state seems consistent) */
|
|
680
|
+
const stats = readStats(path);
|
|
681
|
+
if (stats == null)
|
|
682
|
+
this._cacheManager.drop(path);
|
|
683
|
+
else if (stats[0] == data.byteLength)
|
|
684
|
+
this._cacheManager.add(path, data, stats[1], age);
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
/* simple wrapper function to create a cache */
|
|
688
|
+
export function createCache(config) {
|
|
689
|
+
return new CacheHost(config);
|
|
690
|
+
}
|
|
691
|
+
export class BurntCacheConfig {
|
|
692
|
+
immutableStatePath;
|
|
693
|
+
cacheSize;
|
|
694
|
+
fileSizeLimit;
|
|
695
|
+
alwaysValidate;
|
|
696
|
+
immutableTagging;
|
|
697
|
+
constructor(config) {
|
|
698
|
+
this.immutableStatePath = config?.immutableStatePath ?? '';
|
|
699
|
+
this.cacheSize = config?.cacheSize ?? 50_000_000;
|
|
700
|
+
this.fileSizeLimit = config?.fileSizeLimit ?? 10_000_000;
|
|
701
|
+
this.alwaysValidate = config?.alwaysValidate ?? false;
|
|
702
|
+
this.immutableTagging = config?.immutableTagging ?? true;
|
|
703
|
+
}
|
|
704
|
+
static from(config) {
|
|
705
|
+
return (config instanceof BurntCacheConfig ? config : new BurntCacheConfig(config));
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
//# sourceMappingURL=cache.js.map
|