@e-mc/file-manager 0.5.3 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +347 -92
- package/package.json +10 -10
package/index.js
CHANGED
|
@@ -36,7 +36,7 @@ const CACHE_ETAG = {};
|
|
|
36
36
|
const DISK = {
|
|
37
37
|
ENABLED: false,
|
|
38
38
|
EXPIRES: 0,
|
|
39
|
-
LIMIT: (0, types_1.formatSize)(
|
|
39
|
+
LIMIT: (0, types_1.formatSize)("1gb" /* CACHE_SIZE.FILE_MANAGER_DISK_LIMIT */),
|
|
40
40
|
INCLUDE: [],
|
|
41
41
|
EXCLUDE: []
|
|
42
42
|
};
|
|
@@ -46,8 +46,8 @@ const MEMORY = {
|
|
|
46
46
|
SIZE: 0,
|
|
47
47
|
TOTAL: 0,
|
|
48
48
|
EXPIRES: 0,
|
|
49
|
-
LIMIT: (0, types_1.formatSize)(
|
|
50
|
-
LIMIT_ALL: (0, types_1.formatSize)(
|
|
49
|
+
LIMIT: (0, types_1.formatSize)("100mb" /* CACHE_SIZE.FILE_MANAGER_MEMORY_LIMIT */),
|
|
50
|
+
LIMIT_ALL: (0, types_1.formatSize)("512mb" /* CACHE_SIZE.FILE_MANAGER_MEMORY_LIMIT_ALL */),
|
|
51
51
|
INCLUDE: [],
|
|
52
52
|
EXCLUDE: [],
|
|
53
53
|
PURGE: 0.25,
|
|
@@ -126,6 +126,76 @@ function resetAssets() {
|
|
|
126
126
|
this.reset();
|
|
127
127
|
FileManager.sanitizeAssets(this.assets);
|
|
128
128
|
}
|
|
129
|
+
function recurseDir(output, subDirs, options) {
|
|
130
|
+
const { ignore, sortBy, recursive } = options;
|
|
131
|
+
const items = fs.readdirSync(path.join(...subDirs), { withFileTypes: true })
|
|
132
|
+
.filter(item => item.name)
|
|
133
|
+
.sort((a, b) => {
|
|
134
|
+
if (sortBy > 0) {
|
|
135
|
+
if (a.isDirectory() && !b.isDirectory()) {
|
|
136
|
+
return sortBy & 1 /* READDIR_SORT.DIRECTORY */ ? -1 : 1;
|
|
137
|
+
}
|
|
138
|
+
if (!a.isDirectory() && b.isDirectory()) {
|
|
139
|
+
return sortBy & 1 /* READDIR_SORT.DIRECTORY */ ? 1 : -1;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return (a.name < b.name ? -1 : 1) * (2 /* READDIR_SORT.DESCENDING */ & 2 ? -1 : 1);
|
|
143
|
+
});
|
|
144
|
+
for (const item of items) {
|
|
145
|
+
if (!item.isDirectory()) {
|
|
146
|
+
const pathname = path.join(item.path, item.name);
|
|
147
|
+
if (!ignore.includes(pathname)) {
|
|
148
|
+
output.push(pathname);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
else if (recursive) {
|
|
152
|
+
recurseDir(output, subDirs.concat(item.name), options);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return output;
|
|
156
|
+
}
|
|
157
|
+
function checkHash(data, fromBuffer, options) {
|
|
158
|
+
if (options) {
|
|
159
|
+
let algorithm, digest, value;
|
|
160
|
+
if ((0, types_1.isObject)(options)) {
|
|
161
|
+
({ algorithm, digest, value } = options);
|
|
162
|
+
}
|
|
163
|
+
else {
|
|
164
|
+
value = options;
|
|
165
|
+
}
|
|
166
|
+
if ((0, types_1.isString)(value)) {
|
|
167
|
+
if (!fromBuffer) {
|
|
168
|
+
try {
|
|
169
|
+
data = fs.readFileSync(data);
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
return false;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
return value === core_1.Host.asHash(data, { algorithm, digest });
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return true;
|
|
179
|
+
}
|
|
180
|
+
function filterPaths(values, include, exclude) {
|
|
181
|
+
if (include) {
|
|
182
|
+
if ((0, types_1.isString)(include)) {
|
|
183
|
+
include = [include];
|
|
184
|
+
}
|
|
185
|
+
if ((0, types_1.isArray)(include)) {
|
|
186
|
+
return values.filter(value => core_1.Permission.match(value, include));
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
if (exclude) {
|
|
190
|
+
if ((0, types_1.isString)(exclude)) {
|
|
191
|
+
exclude = [exclude];
|
|
192
|
+
}
|
|
193
|
+
if ((0, types_1.isArray)(exclude)) {
|
|
194
|
+
return values.filter(value => !core_1.Permission.match(value, exclude));
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
return values;
|
|
198
|
+
}
|
|
129
199
|
function abortedHost() {
|
|
130
200
|
if (this.finalizeState === 6 /* FINALIZE_STATE.ABORTED */) {
|
|
131
201
|
this.restarting = false;
|
|
@@ -195,6 +265,8 @@ function collectErrors() {
|
|
|
195
265
|
function rejectModule(err, type, hint) {
|
|
196
266
|
this.writeFail(["Handled rejection" /* ERR_MESSAGE.HANDLED_REJECTION */, this.moduleName + (hint ? ': ' + hint : '')], err, type);
|
|
197
267
|
}
|
|
268
|
+
const checksumFile = (algorithm) => "checksum" /* HASH_OUTPUT.FILENAME */ + '.' + ((0, types_1.isString)(algorithm) ? algorithm.toLowerCase() : "sha256" /* HASH_OUTPUT.ALGORITHM */);
|
|
269
|
+
const checksumError = (algorithm) => new Error("Invalid parameters" /* ERR_MESSAGE.PARAMETERS */ + ` (${algorithm || "sha256" /* HASH_OUTPUT.ALGORITHM */})`);
|
|
198
270
|
const isFunction = (value) => typeof value === 'function';
|
|
199
271
|
const ignoreAsset = (item, exists) => item.invalid || (0, types_1.hasBit)(item.flags, 1 /* ASSET_FLAG.IGNORE */ | (!exists ? 128 /* ASSET_FLAG.EXISTS */ : 0));
|
|
200
272
|
class HttpDiskCache {
|
|
@@ -557,6 +629,146 @@ class FileManager extends core_1.Host {
|
|
|
557
629
|
});
|
|
558
630
|
return assets;
|
|
559
631
|
}
|
|
632
|
+
static async writeChecksum(root, to, options) {
|
|
633
|
+
if ((0, types_1.isObject)(to)) {
|
|
634
|
+
options = to;
|
|
635
|
+
to = undefined;
|
|
636
|
+
}
|
|
637
|
+
else {
|
|
638
|
+
options || (options = {});
|
|
639
|
+
}
|
|
640
|
+
const { algorithm, digest, sortBy = 0, recursive = false, ignore = [], include, exclude, verbose = false, joinRoot } = options;
|
|
641
|
+
to || (to = checksumFile(algorithm));
|
|
642
|
+
let result = [];
|
|
643
|
+
try {
|
|
644
|
+
const filename = path.basename(to);
|
|
645
|
+
to = joinRoot ? path.join(root, to) : path.resolve(to);
|
|
646
|
+
recurseDir(result, [root], { ignore: [...ignore, to], sortBy, recursive });
|
|
647
|
+
const output = [];
|
|
648
|
+
for (const pathname of result = filterPaths(result, include, exclude)) {
|
|
649
|
+
if (recursive === 1 && path.basename(pathname) === filename) {
|
|
650
|
+
continue;
|
|
651
|
+
}
|
|
652
|
+
const current = await this.readHash(pathname, { algorithm, digest }) + ' ' + (path.sep === '\\' ? pathname.replace(/\\/g, '/') : pathname).substring(root.length).replace(/^\//, '');
|
|
653
|
+
if (verbose) {
|
|
654
|
+
process.stdout.write(current + '\n');
|
|
655
|
+
}
|
|
656
|
+
output.push(current);
|
|
657
|
+
}
|
|
658
|
+
if (output.length) {
|
|
659
|
+
fs.writeFileSync(to, output.join('\n'), 'utf-8');
|
|
660
|
+
options.outPath = to;
|
|
661
|
+
}
|
|
662
|
+
else {
|
|
663
|
+
if (this.isPath(to)) {
|
|
664
|
+
fs.unlinkSync(to);
|
|
665
|
+
}
|
|
666
|
+
if (options.throwsEmpty) {
|
|
667
|
+
throw checksumError(algorithm);
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
catch (err) {
|
|
672
|
+
if (options.throwsEmpty) {
|
|
673
|
+
throw err;
|
|
674
|
+
}
|
|
675
|
+
this.writeFail(["Unable to read directory" /* ERR_MESSAGE.READ_DIRECTORY */, root], err, 32 /* LOG_TYPE.FILE */);
|
|
676
|
+
return null;
|
|
677
|
+
}
|
|
678
|
+
return result;
|
|
679
|
+
}
|
|
680
|
+
static async verifyChecksum(root, from, options) {
|
|
681
|
+
if ((0, types_1.isObject)(from)) {
|
|
682
|
+
options = from;
|
|
683
|
+
from = undefined;
|
|
684
|
+
}
|
|
685
|
+
else {
|
|
686
|
+
options || (options = {});
|
|
687
|
+
}
|
|
688
|
+
const { algorithm, digest, sortBy = 0, recursive = false, ignore = [], include, exclude, verbose = true, joinRoot } = options;
|
|
689
|
+
const parent = recursive === 1 && typeof verbose !== 'number';
|
|
690
|
+
from || (from = checksumFile(algorithm));
|
|
691
|
+
let fail = [], missing = [];
|
|
692
|
+
try {
|
|
693
|
+
const filename = path.basename(from);
|
|
694
|
+
const files = [];
|
|
695
|
+
from = joinRoot ? path.join(root, filename) : path.resolve(from);
|
|
696
|
+
recurseDir(files, [root], { ignore: [...ignore, from], sortBy, recursive });
|
|
697
|
+
const items = fs.readFileSync(from, 'utf-8').split('\n').map(item => {
|
|
698
|
+
const index = item.indexOf(' ');
|
|
699
|
+
return [item.substring(0, index), path.join(root, item.substring(index + 1))];
|
|
700
|
+
});
|
|
701
|
+
const checked = include || exclude ? filterPaths(items.map(item => item[1]), include, exclude) : null;
|
|
702
|
+
let valid;
|
|
703
|
+
for (const [previous, pathname] of items) {
|
|
704
|
+
if (checked !== null && !checked.includes(pathname) || recursive === 1 && path.basename(pathname) === filename) {
|
|
705
|
+
continue;
|
|
706
|
+
}
|
|
707
|
+
if (files.includes(pathname)) {
|
|
708
|
+
const hash = await this.readHash(pathname, { algorithm, digest });
|
|
709
|
+
if (hash !== previous) {
|
|
710
|
+
fail.push(pathname);
|
|
711
|
+
}
|
|
712
|
+
else if (verbose) {
|
|
713
|
+
process.stdout.write("+" /* SUMDIR_STATUS.PASS */ + ' ' + pathname + '\n');
|
|
714
|
+
}
|
|
715
|
+
valid = true;
|
|
716
|
+
}
|
|
717
|
+
else if (!ignore.includes(pathname)) {
|
|
718
|
+
missing.push(pathname);
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
if (parent) {
|
|
722
|
+
const nested = files.filter(value => path.basename(value) === filename);
|
|
723
|
+
if (nested.length) {
|
|
724
|
+
const current = items.map(item => item[1]);
|
|
725
|
+
const tasks = nested.map(pathname => this.verifyChecksum(path.dirname(pathname), filename, { ...options, ignore: current, joinRoot: true, verbose: (verbose ? 1 : 0) }));
|
|
726
|
+
await Promise.all(tasks).then(group => {
|
|
727
|
+
for (const item of group) {
|
|
728
|
+
if (item) {
|
|
729
|
+
const [f, m] = item;
|
|
730
|
+
if (f.length) {
|
|
731
|
+
fail.push(...f);
|
|
732
|
+
}
|
|
733
|
+
if (m.length) {
|
|
734
|
+
missing.push(...m);
|
|
735
|
+
}
|
|
736
|
+
valid = true;
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
});
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
if (valid) {
|
|
743
|
+
options.outPath = from;
|
|
744
|
+
}
|
|
745
|
+
else if (options.throwsEmpty) {
|
|
746
|
+
throw checksumError(algorithm);
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
catch (err) {
|
|
750
|
+
if (options.throwsEmpty) {
|
|
751
|
+
throw err;
|
|
752
|
+
}
|
|
753
|
+
this.writeFail(["Unable to read directory" /* ERR_MESSAGE.READ_DIRECTORY */, root], err, 32 /* LOG_TYPE.FILE */);
|
|
754
|
+
return null;
|
|
755
|
+
}
|
|
756
|
+
if (parent) {
|
|
757
|
+
if (fail.length) {
|
|
758
|
+
fail = Array.from(new Set(fail));
|
|
759
|
+
}
|
|
760
|
+
if (missing.length) {
|
|
761
|
+
missing = Array.from(new Set(missing));
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
if (verbose === true) {
|
|
765
|
+
const max = Math.max(...fail.concat(missing).map(item => item.length));
|
|
766
|
+
const writeLog = (items, symbol) => items.forEach((value, index) => process.stdout.write(`${symbol} ${value.padEnd(max)} (${(index + 1).toString()})\n`));
|
|
767
|
+
writeLog(fail, "-" /* SUMDIR_STATUS.FAIL */);
|
|
768
|
+
writeLog(missing, "?" /* SUMDIR_STATUS.MISSING */);
|
|
769
|
+
}
|
|
770
|
+
return [fail, missing];
|
|
771
|
+
}
|
|
560
772
|
static createFileThread(host, file) {
|
|
561
773
|
return new FileThread(host, file, 0);
|
|
562
774
|
}
|
|
@@ -2242,22 +2454,19 @@ class FileManager extends core_1.Host {
|
|
|
2242
2454
|
bundling[_l = file.uri] || (bundling[_l] = []);
|
|
2243
2455
|
if (bundleIndex > 0) {
|
|
2244
2456
|
items[bundleIndex - 1] = file;
|
|
2245
|
-
|
|
2246
|
-
if (file.fetchType === 1 /* FETCH_TYPE.HTTP */ && (cacheToDisk.has(url) || cacheToMemory.has(url))) {
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
.on('timeout', () => resolve(file));
|
|
2259
|
-
}));
|
|
2260
|
-
}
|
|
2457
|
+
let url, parent;
|
|
2458
|
+
if (file.fetchType === 1 /* FETCH_TYPE.HTTP */ && (cacheToDisk.has(url = file.url) || cacheToMemory.has(url)) && (parent = assets.find(item => item.bundleIndex === 0 && item.bundleId === bundleId))) {
|
|
2459
|
+
(parent.bundleQueue || (parent.bundleQueue = [])).push(new Promise(resolve => {
|
|
2460
|
+
this.Request.open(url, { method: 'HEAD', httpVersion: 1 })
|
|
2461
|
+
.on('response', res => {
|
|
2462
|
+
if (res.statusCode < 300 /* HTTP_STATUS.MULTIPLE_CHOICES */) {
|
|
2463
|
+
applyHeaders(file, res.headers);
|
|
2464
|
+
}
|
|
2465
|
+
resolve(file);
|
|
2466
|
+
})
|
|
2467
|
+
.on('error', () => resolve(file))
|
|
2468
|
+
.on('timeout', () => resolve(file));
|
|
2469
|
+
}));
|
|
2261
2470
|
}
|
|
2262
2471
|
return true;
|
|
2263
2472
|
}
|
|
@@ -2322,9 +2531,10 @@ class FileManager extends core_1.Host {
|
|
|
2322
2531
|
completed[localUri] = file;
|
|
2323
2532
|
if (!(0, types_1.isEmpty)(file.bundleId) && file.bundleIndex === 0) {
|
|
2324
2533
|
this.setAssetContent(file, this.getUTF8String(file, localUri));
|
|
2534
|
+
const bundleQueue = file.bundleQueue;
|
|
2325
2535
|
let success = true, error, checkEtag = !file.trailingContent && hasEtag(file);
|
|
2326
|
-
if (
|
|
2327
|
-
error = await Promise.all(
|
|
2536
|
+
if (bundleQueue) {
|
|
2537
|
+
error = await Promise.all(bundleQueue).then(() => null).catch((err) => err);
|
|
2328
2538
|
delete file.bundleQueue;
|
|
2329
2539
|
}
|
|
2330
2540
|
const items = appending[localUri];
|
|
@@ -2584,6 +2794,10 @@ class FileManager extends core_1.Host {
|
|
|
2584
2794
|
return true;
|
|
2585
2795
|
};
|
|
2586
2796
|
const fileReceived = (item, localUri, err, fetched, binary) => {
|
|
2797
|
+
if (item.checksum && !err && !checkHash(item.buffer || localUri, !!item.buffer, item.checksum)) {
|
|
2798
|
+
err = (0, types_1.errorValue)("Checksum did not match" /* ERR_MESSAGE.FAILED_CHECKSUM */, item.uri);
|
|
2799
|
+
this.filesToRemove.add(localUri);
|
|
2800
|
+
}
|
|
2587
2801
|
if (err) {
|
|
2588
2802
|
item.invalid = true;
|
|
2589
2803
|
this.completeAsyncTask(err, localUri);
|
|
@@ -2607,8 +2821,9 @@ class FileManager extends core_1.Host {
|
|
|
2607
2821
|
if (item.invalid || (0, types_1.ignoreFlag)(item.flags)) {
|
|
2608
2822
|
continue;
|
|
2609
2823
|
}
|
|
2610
|
-
|
|
2611
|
-
|
|
2824
|
+
let { filename, uri } = item;
|
|
2825
|
+
if (!filename) {
|
|
2826
|
+
if (!(uri && core_1.Host.isFile(uri, 'torrent'))) {
|
|
2612
2827
|
item.invalid = true;
|
|
2613
2828
|
continue;
|
|
2614
2829
|
}
|
|
@@ -2640,13 +2855,13 @@ class FileManager extends core_1.Host {
|
|
|
2640
2855
|
item.incremental = undefined;
|
|
2641
2856
|
}
|
|
2642
2857
|
}
|
|
2643
|
-
if (valid &&
|
|
2644
|
-
let uri = item.uri, cached;
|
|
2858
|
+
if (valid && filename) {
|
|
2645
2859
|
const etag = (cacheEtag && type !== 'exists' || type === 'etag') && !!uri;
|
|
2646
2860
|
if (etag && imported.some(file => file.imported.includes(uri))) {
|
|
2647
2861
|
item.incremental = 'none';
|
|
2648
2862
|
}
|
|
2649
2863
|
else if (!(0, types_1.mainFlag)(item.flags)) {
|
|
2864
|
+
let cached;
|
|
2650
2865
|
const { bundleId, bundleIndex = -1 } = item;
|
|
2651
2866
|
const setBuffer = (target) => {
|
|
2652
2867
|
if (!etag && (uri = target.uri) && (cached = MEMORY.CACHE[uri])) {
|
|
@@ -2691,7 +2906,7 @@ class FileManager extends core_1.Host {
|
|
|
2691
2906
|
if (!cached) {
|
|
2692
2907
|
setBuffer(item);
|
|
2693
2908
|
}
|
|
2694
|
-
if (core_1.Host.isPath(localUri)) {
|
|
2909
|
+
if (core_1.Host.isPath(localUri) && (!item.checksumOutput || checkHash(localUri, false, item.checksumOutput))) {
|
|
2695
2910
|
item.flags |= 128 /* ASSET_FLAG.EXISTS */;
|
|
2696
2911
|
if (!etag || item.fetchType !== 1 /* FETCH_TYPE.HTTP */) {
|
|
2697
2912
|
if (!(0, types_1.isEmpty)(bundleId) && bundleIndex === 0) {
|
|
@@ -2709,13 +2924,15 @@ class FileManager extends core_1.Host {
|
|
|
2709
2924
|
else if (cached) {
|
|
2710
2925
|
try {
|
|
2711
2926
|
const buffer = cached[1];
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
item.
|
|
2927
|
+
if (!item.checksum || checkHash(buffer, true, item.checksum)) {
|
|
2928
|
+
fs.writeFileSync(localUri, buffer);
|
|
2929
|
+
if (item.willChange && Buffer.isBuffer(buffer)) {
|
|
2930
|
+
item.buffer = buffer;
|
|
2931
|
+
}
|
|
2932
|
+
this.performAsyncTask();
|
|
2933
|
+
fileReceived(item, localUri, null, true);
|
|
2934
|
+
continue;
|
|
2715
2935
|
}
|
|
2716
|
-
this.performAsyncTask();
|
|
2717
|
-
fileReceived(item, localUri, null, true);
|
|
2718
|
-
continue;
|
|
2719
2936
|
}
|
|
2720
2937
|
catch {
|
|
2721
2938
|
}
|
|
@@ -2726,13 +2943,12 @@ class FileManager extends core_1.Host {
|
|
|
2726
2943
|
try {
|
|
2727
2944
|
const buffer = fs.readFileSync(localUri);
|
|
2728
2945
|
if (!etag) {
|
|
2729
|
-
(0, asset_1.setInitialValue)(item, false);
|
|
2730
|
-
item.initialValue.buffer = buffer;
|
|
2946
|
+
(0, asset_1.setInitialValue)(item, false).buffer = buffer;
|
|
2731
2947
|
}
|
|
2732
2948
|
else if (uri && CACHE_ETAG[uri]) {
|
|
2733
|
-
(0, asset_1.setInitialValue)(item, false);
|
|
2734
|
-
|
|
2735
|
-
|
|
2949
|
+
const initialValue = (0, asset_1.setInitialValue)(item, false);
|
|
2950
|
+
initialValue.buffer = buffer;
|
|
2951
|
+
initialValue.etag = CACHE_ETAG[uri];
|
|
2736
2952
|
}
|
|
2737
2953
|
}
|
|
2738
2954
|
catch {
|
|
@@ -2753,12 +2969,12 @@ class FileManager extends core_1.Host {
|
|
|
2753
2969
|
break;
|
|
2754
2970
|
}
|
|
2755
2971
|
const { pathname, localUri } = download;
|
|
2756
|
-
let uri;
|
|
2757
2972
|
if (item.content) {
|
|
2758
2973
|
if (!checkQueue(item, localUri, pathname, true)) {
|
|
2759
|
-
|
|
2974
|
+
const content = item.content;
|
|
2975
|
+
item.sourceUTF8 = content;
|
|
2760
2976
|
this.performAsyncTask();
|
|
2761
|
-
fs.writeFile(localUri,
|
|
2977
|
+
fs.writeFile(localUri, content, item.encoding || (item.encoding = 'utf-8'), err => fileReceived(item, localUri, err));
|
|
2762
2978
|
}
|
|
2763
2979
|
}
|
|
2764
2980
|
else if (item.dataView) {
|
|
@@ -2779,18 +2995,22 @@ class FileManager extends core_1.Host {
|
|
|
2779
2995
|
fs.writeFile(localUri, item.base64, 'base64', err => fileReceived(item, localUri, err, false, true));
|
|
2780
2996
|
}
|
|
2781
2997
|
}
|
|
2782
|
-
else
|
|
2998
|
+
else {
|
|
2999
|
+
const uri = item.uri;
|
|
3000
|
+
if (!uri) {
|
|
3001
|
+
item.invalid = true;
|
|
3002
|
+
continue;
|
|
3003
|
+
}
|
|
2783
3004
|
const checkDest = (src) => staging || !(0, lib_v4_1.hasSameStat)(src, localUri);
|
|
2784
|
-
const type = item.fetchType
|
|
3005
|
+
const type = item.fetchType || 0 /* FETCH_TYPE.UNKNOWN */;
|
|
2785
3006
|
const bundleMain = !(0, types_1.isEmpty)(item.bundleId) && item.bundleIndex === 0;
|
|
2786
|
-
|
|
2787
|
-
if (
|
|
2788
|
-
|
|
2789
|
-
const checkEtag = hasEtag(item);
|
|
3007
|
+
const isHttp = type === 1 /* FETCH_TYPE.HTTP */;
|
|
3008
|
+
if (isHttp || type === 2 /* FETCH_TYPE.UNIX_SOCKET */) {
|
|
3009
|
+
let checkEtag;
|
|
2790
3010
|
if ((0, types_1.isEmpty)(item.bundleId) && bundling[uri]) {
|
|
2791
3011
|
bundling[uri].push(item);
|
|
2792
3012
|
}
|
|
2793
|
-
else if (checkEtag || !checkQueue(item, localUri, pathname)) {
|
|
3013
|
+
else if ((checkEtag = hasEtag(item)) || !checkQueue(item, localUri, pathname)) {
|
|
2794
3014
|
if (!isHttp && !checkDest(uri)) {
|
|
2795
3015
|
fileReceived(item, localUri);
|
|
2796
3016
|
}
|
|
@@ -2804,6 +3024,7 @@ class FileManager extends core_1.Host {
|
|
|
2804
3024
|
if (!checkEtag) {
|
|
2805
3025
|
downloading[uri] = [];
|
|
2806
3026
|
}
|
|
3027
|
+
const url = item.url;
|
|
2807
3028
|
let cacheDir, cacheBuffer, mainEtag, encoding = item.encoding, client = null;
|
|
2808
3029
|
if (isHttp) {
|
|
2809
3030
|
cacheDir = cacheToDisk.has(url) && isCacheable(item);
|
|
@@ -3078,9 +3299,6 @@ class FileManager extends core_1.Host {
|
|
|
3078
3299
|
}
|
|
3079
3300
|
}
|
|
3080
3301
|
}
|
|
3081
|
-
else {
|
|
3082
|
-
item.invalid = true;
|
|
3083
|
-
}
|
|
3084
3302
|
}
|
|
3085
3303
|
this.cleared = true;
|
|
3086
3304
|
}
|
|
@@ -3091,35 +3309,37 @@ class FileManager extends core_1.Host {
|
|
|
3091
3309
|
}
|
|
3092
3310
|
const tasks = [];
|
|
3093
3311
|
for (const item of assets) {
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
|
|
3104
|
-
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
|
|
3110
|
-
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
item.buffer = result;
|
|
3117
|
-
}
|
|
3118
|
-
})
|
|
3119
|
-
.catch(err => this.writeFail(["Unable to compress file" /* ERR_MESSAGE.COMPRESS_FILE */, path.basename(file)], err, { type: 8 /* LOG_TYPE.COMPRESS */, startTime: options.startTime })));
|
|
3120
|
-
}
|
|
3312
|
+
const localUri = item.localUri;
|
|
3313
|
+
if (!(localUri && image_1.default.isBinary(item.mimeType))) {
|
|
3314
|
+
continue;
|
|
3315
|
+
}
|
|
3316
|
+
const files = [localUri];
|
|
3317
|
+
if (item.transforms) {
|
|
3318
|
+
files.push(...item.transforms);
|
|
3319
|
+
}
|
|
3320
|
+
const format = item.mimeType.split('/').pop();
|
|
3321
|
+
for (const file of files) {
|
|
3322
|
+
const output = item.compress.filter(({ condition }) => !condition || withinSizeRange(file, condition));
|
|
3323
|
+
if (!(output.length && this.has(file))) {
|
|
3324
|
+
continue;
|
|
3325
|
+
}
|
|
3326
|
+
const found = output.filter(value => value.format === format);
|
|
3327
|
+
for (const config of found.length ? found : output) {
|
|
3328
|
+
const options = {
|
|
3329
|
+
...config,
|
|
3330
|
+
timeout: this[kProcessTimeout].compress ?? PROCESS_TIMEOUT.compress,
|
|
3331
|
+
proxyUrl: (uri) => {
|
|
3332
|
+
const proxy = this.Request.proxyOf(uri);
|
|
3333
|
+
return proxy ? proxy.host.href : '';
|
|
3121
3334
|
}
|
|
3122
|
-
}
|
|
3335
|
+
};
|
|
3336
|
+
tasks.push(compress.tryImage(item.buffer && files.length === 1 ? item.buffer : file, file, options)
|
|
3337
|
+
.then(result => {
|
|
3338
|
+
if (result instanceof Buffer && file === localUri) {
|
|
3339
|
+
item.buffer = result;
|
|
3340
|
+
}
|
|
3341
|
+
})
|
|
3342
|
+
.catch(err => this.writeFail(["Unable to compress file" /* ERR_MESSAGE.COMPRESS_FILE */, path.basename(file)], err, { type: 8 /* LOG_TYPE.COMPRESS */, startTime: options.startTime })));
|
|
3123
3343
|
}
|
|
3124
3344
|
}
|
|
3125
3345
|
}
|
|
@@ -3132,7 +3352,7 @@ class FileManager extends core_1.Host {
|
|
|
3132
3352
|
if (constructor.finalize && instance.assets.length) {
|
|
3133
3353
|
await constructor.finalize.call(this, instance).catch(err => rejectModule.call(instance, err, 4 /* LOG_TYPE.PROCESS */));
|
|
3134
3354
|
if (this.aborted) {
|
|
3135
|
-
|
|
3355
|
+
return;
|
|
3136
3356
|
}
|
|
3137
3357
|
}
|
|
3138
3358
|
}
|
|
@@ -3144,25 +3364,17 @@ class FileManager extends core_1.Host {
|
|
|
3144
3364
|
if (items.length) {
|
|
3145
3365
|
await (constructor.using || constructor.finalize).call(this, instance, items).catch(err => rejectModule.call(instance, err, 4 /* LOG_TYPE.PROCESS */));
|
|
3146
3366
|
if (this.aborted) {
|
|
3147
|
-
|
|
3367
|
+
return;
|
|
3148
3368
|
}
|
|
3149
3369
|
}
|
|
3150
3370
|
}
|
|
3151
3371
|
}
|
|
3152
3372
|
}
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
|
|
3156
|
-
}
|
|
3373
|
+
finalizeCloud() {
|
|
3374
|
+
const cloud = this.Cloud;
|
|
3375
|
+
return cloud ? cloud_1.default.finalize.call(this, cloud).catch(err => rejectModule.call(cloud, err, 64 /* LOG_TYPE.CLOUD */)) : Promise.resolve();
|
|
3157
3376
|
}
|
|
3158
3377
|
async finalizeCleanup() {
|
|
3159
|
-
if (this.Compress) {
|
|
3160
|
-
const tasks = [];
|
|
3161
|
-
this.assets.forEach(item => item.compress && !ignoreAsset(item, true) && tasks.push(this.compressFile(item, false)));
|
|
3162
|
-
if (tasks.length) {
|
|
3163
|
-
await Promise.allSettled(tasks);
|
|
3164
|
-
}
|
|
3165
|
-
}
|
|
3166
3378
|
if (this.emptyDir.size) {
|
|
3167
3379
|
for (const value of Array.from(this.emptyDir).reverse()) {
|
|
3168
3380
|
try {
|
|
@@ -3236,11 +3448,54 @@ class FileManager extends core_1.Host {
|
|
|
3236
3448
|
return Promise.reject((0, types_1.createAbortError)());
|
|
3237
3449
|
}
|
|
3238
3450
|
removeFiles();
|
|
3451
|
+
for (const item of this.assets) {
|
|
3452
|
+
if (item.checksumOutput && !ignoreAsset(item)) {
|
|
3453
|
+
const localUri = item.localUri;
|
|
3454
|
+
if (!checkHash(localUri, false, item.checksumOutput)) {
|
|
3455
|
+
item.invalid = true;
|
|
3456
|
+
filesToRemove.add(localUri);
|
|
3457
|
+
this.writeFail(["Invalid checksum" /* ERR_MESSAGE.CHECKSUM */, path.basename(localUri)], (0, types_1.errorValue)("Checksum did not match" /* ERR_MESSAGE.FAILED_CHECKSUM */, localUri), { type: 32 /* LOG_TYPE.FILE */, startTime });
|
|
3458
|
+
}
|
|
3459
|
+
}
|
|
3460
|
+
}
|
|
3461
|
+
removeFiles();
|
|
3239
3462
|
await this.finalizeCloud().catch(err => rejectModule.call(this, err, 64 /* LOG_TYPE.CLOUD */));
|
|
3240
3463
|
if (this.aborted) {
|
|
3241
3464
|
return Promise.reject((0, types_1.createAbortError)());
|
|
3242
3465
|
}
|
|
3243
3466
|
removeFiles();
|
|
3467
|
+
if (this.Compress) {
|
|
3468
|
+
const tasks = [];
|
|
3469
|
+
this.assets.forEach(item => item.compress && !ignoreAsset(item, true) && tasks.push(this.compressFile(item, false)));
|
|
3470
|
+
if (tasks.length) {
|
|
3471
|
+
await Promise.allSettled(tasks);
|
|
3472
|
+
}
|
|
3473
|
+
}
|
|
3474
|
+
let checksum = this.config.checksum;
|
|
3475
|
+
if (checksum) {
|
|
3476
|
+
if (typeof checksum === 'boolean' || checksum === 1) {
|
|
3477
|
+
checksum = { recursive: checksum };
|
|
3478
|
+
}
|
|
3479
|
+
else if ((0, types_1.isString)(checksum)) {
|
|
3480
|
+
const items = checksum.split('.');
|
|
3481
|
+
checksum = items.length > 1 ? { algorithm: items[items.length - 1], filename: checksum } : { algorithm: checksum };
|
|
3482
|
+
}
|
|
3483
|
+
if ((0, types_1.isPlainObject)(checksum)) {
|
|
3484
|
+
const baseDirectory = this.baseDirectory;
|
|
3485
|
+
checksum.joinRoot = true;
|
|
3486
|
+
checksum.throwsEmpty = true;
|
|
3487
|
+
const sumTime = LOG_TIMEELAPSED ? process.hrtime() : 0;
|
|
3488
|
+
try {
|
|
3489
|
+
const files = (await FileManager.writeChecksum(baseDirectory, checksum.filename, checksum));
|
|
3490
|
+
if (sumTime) {
|
|
3491
|
+
this.writeTimeElapsed(checksum.algorithm || "sha256" /* HASH_OUTPUT.ALGORITHM */, [baseDirectory, files.length + (files.length === 1 ? ' file' : ' files')], sumTime, { ...core_1.Host.LOG_STYLE_WARN });
|
|
3492
|
+
}
|
|
3493
|
+
}
|
|
3494
|
+
catch (err) {
|
|
3495
|
+
this.writeFail(["Unable to read directory" /* ERR_MESSAGE.READ_DIRECTORY */, path.basename(baseDirectory)], err, { type: 32 /* LOG_TYPE.FILE */, startTime });
|
|
3496
|
+
}
|
|
3497
|
+
}
|
|
3498
|
+
}
|
|
3244
3499
|
await this.finalizeCleanup().catch(err => rejectModule.call(this, err, 1 /* LOG_TYPE.SYSTEM */));
|
|
3245
3500
|
removeFiles();
|
|
3246
3501
|
if (LOG_TIMEELAPSED) {
|
|
@@ -3248,7 +3503,7 @@ class FileManager extends core_1.Host {
|
|
|
3248
3503
|
const errorCount = this.errorCount;
|
|
3249
3504
|
const message = [];
|
|
3250
3505
|
if (errorCount) {
|
|
3251
|
-
message.push(
|
|
3506
|
+
message.push('ERROR ' + errorCount); // eslint-disable-line @typescript-eslint/no-base-to-string
|
|
3252
3507
|
}
|
|
3253
3508
|
let [size, count] = h;
|
|
3254
3509
|
if (count) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@e-mc/file-manager",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"description": "FileManager constructor for E-mc.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"types": "index.d.ts",
|
|
@@ -20,14 +20,14 @@
|
|
|
20
20
|
"license": "BSD 3-Clause",
|
|
21
21
|
"homepage": "https://github.com/anpham6/e-mc#readme",
|
|
22
22
|
"dependencies": {
|
|
23
|
-
"@e-mc/cloud": "0.
|
|
24
|
-
"@e-mc/compress": "0.
|
|
25
|
-
"@e-mc/core": "0.
|
|
26
|
-
"@e-mc/document": "0.
|
|
27
|
-
"@e-mc/image": "0.
|
|
28
|
-
"@e-mc/request": "0.
|
|
29
|
-
"@e-mc/task": "0.
|
|
30
|
-
"@e-mc/types": "0.
|
|
31
|
-
"@e-mc/watch": "0.
|
|
23
|
+
"@e-mc/cloud": "0.7.0",
|
|
24
|
+
"@e-mc/compress": "0.7.0",
|
|
25
|
+
"@e-mc/core": "0.7.0",
|
|
26
|
+
"@e-mc/document": "0.7.0",
|
|
27
|
+
"@e-mc/image": "0.7.0",
|
|
28
|
+
"@e-mc/request": "0.7.0",
|
|
29
|
+
"@e-mc/task": "0.7.0",
|
|
30
|
+
"@e-mc/types": "0.7.0",
|
|
31
|
+
"@e-mc/watch": "0.7.0"
|
|
32
32
|
}
|
|
33
33
|
}
|