@e-mc/file-manager 0.8.6 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +10 -10
- package/README.md +149 -36
- package/index.d.ts +5 -5
- package/index.js +595 -271
- package/package.json +12 -11
package/index.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
|
|
3
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
2
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m;
|
|
4
3
|
const path = require("path");
|
|
5
4
|
const fs = require("fs");
|
|
5
|
+
const stream = require("stream");
|
|
6
6
|
const pm = require("picomatch");
|
|
7
|
-
const
|
|
7
|
+
const chalk = require("chalk");
|
|
8
8
|
const util_1 = require("@e-mc/request/util");
|
|
9
9
|
const asset_1 = require("@e-mc/document/asset");
|
|
10
10
|
const util_2 = require("@e-mc/document/util");
|
|
@@ -22,15 +22,19 @@ const kIncremental = Symbol('incremental');
|
|
|
22
22
|
const kRestarting = Symbol('restarting');
|
|
23
23
|
const kDelayed = Symbol('delayed');
|
|
24
24
|
const kCleared = Symbol('cleared');
|
|
25
|
+
const kFinalizedState = Symbol('finalizeState');
|
|
25
26
|
const KTimerMain = Symbol('timerMain');
|
|
26
27
|
const KEnabled = Symbol('enabled');
|
|
27
28
|
const kRecursionLimit = Symbol('recursionLimit');
|
|
28
29
|
const kHost = Symbol('host');
|
|
29
30
|
const kQueuedTasks = Symbol('queuedTasks');
|
|
31
|
+
const kScheduler = Symbol('scheduler');
|
|
30
32
|
const kProcessTimeout = Symbol('processTimeout');
|
|
31
33
|
const kDownloadStats = Symbol('downloadStats');
|
|
32
34
|
const kReplaceMap = Symbol('replaceMap');
|
|
33
35
|
const PLATFORM_WIN32 = process.platform === 'win32';
|
|
36
|
+
const PROCESS_STDOUT = process.stdout;
|
|
37
|
+
const STDOUT_CURSOR = typeof process.stdout.moveCursor === 'function';
|
|
34
38
|
const PROCESS_TIMEOUT = { filemanager: 0, compress: 0 };
|
|
35
39
|
const CACHE_ETAG = {};
|
|
36
40
|
const DISK = {
|
|
@@ -58,8 +62,10 @@ let SESSION_ID = 0;
|
|
|
58
62
|
let SESSION_LIMIT = 1000;
|
|
59
63
|
let ASSET_ID = 0;
|
|
60
64
|
let RECURSION_LIMIT = 10;
|
|
65
|
+
let PROCESS_SUB_LIMIT = Math.max(require('os').cpus().length, 1) * 2;
|
|
61
66
|
let LOG_TIMEELAPSED = true;
|
|
62
67
|
let LOG_TIMEPROCESS = true;
|
|
68
|
+
let LOG_FORMAT = null;
|
|
63
69
|
const HTTP_CLIENT = {
|
|
64
70
|
timeout: 60000,
|
|
65
71
|
connectTimeout: 20 * 1000,
|
|
@@ -88,7 +94,7 @@ function withinSizeRange(uri, value, pattern) {
|
|
|
88
94
|
const match = (pattern || /\(\s*(\d+)\s*(?:,\s*(\d+|\*)\s*)?\)/).exec(value);
|
|
89
95
|
const [minSize, maxSize] = match && !match[1] ? [+match[2], !match[3] || match[3] === '*' ? Infinity : +match[3]] : [0, Infinity];
|
|
90
96
|
if (minSize > 0 || maxSize < Infinity) {
|
|
91
|
-
const fileSize = (0,
|
|
97
|
+
const fileSize = (0, util_1.getSize)(uri);
|
|
92
98
|
if (fileSize === 0 || fileSize < minSize || fileSize > maxSize) {
|
|
93
99
|
return false;
|
|
94
100
|
}
|
|
@@ -99,14 +105,15 @@ function startMessage() {
|
|
|
99
105
|
this.formatMessage(128, 'START', [new Date().toLocaleString(), this.assets.length + ' assets'], this.baseDirectory, { ...core_1.Host.LOG_STYLE_SUCCESS });
|
|
100
106
|
}
|
|
101
107
|
function clearAssets() {
|
|
102
|
-
this.assets.forEach(item =>
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
108
|
+
this.assets.forEach(item => unsetContent(item));
|
|
109
|
+
}
|
|
110
|
+
function unsetContent(item) {
|
|
111
|
+
if ('buffer' in item) {
|
|
112
|
+
item.buffer = null;
|
|
113
|
+
}
|
|
114
|
+
if ('sourceUTF8' in item) {
|
|
115
|
+
item.sourceUTF8 = '';
|
|
116
|
+
}
|
|
110
117
|
}
|
|
111
118
|
function bundleTorrent(files, mimeType, encoding) {
|
|
112
119
|
let output = '';
|
|
@@ -169,7 +176,7 @@ function checkHash(localUri, output, options, data) {
|
|
|
169
176
|
data = fs.readFileSync(localUri);
|
|
170
177
|
}
|
|
171
178
|
catch (err) {
|
|
172
|
-
if (output && err
|
|
179
|
+
if (output && core_1.Host.isErrorCode(err, 'ENOENT')) {
|
|
173
180
|
this.addLog(this.statusType.WARN, algorithm + ' -> ENOENT -> No checksum performed', path.basename(localUri), value);
|
|
174
181
|
return true;
|
|
175
182
|
}
|
|
@@ -223,6 +230,30 @@ function abortedHost() {
|
|
|
223
230
|
}
|
|
224
231
|
return false;
|
|
225
232
|
}
|
|
233
|
+
function formatMinutes(elapsed) {
|
|
234
|
+
let result = '';
|
|
235
|
+
if (elapsed >= 3600000) {
|
|
236
|
+
const h = Math.floor(elapsed / 3600000);
|
|
237
|
+
elapsed -= h * 3600000;
|
|
238
|
+
result = h + ':';
|
|
239
|
+
}
|
|
240
|
+
if (elapsed >= 60000) {
|
|
241
|
+
const m = Math.floor(elapsed / 60000);
|
|
242
|
+
elapsed -= m * 60000;
|
|
243
|
+
result += padStart(m, result ? '0' : ' ') + ':';
|
|
244
|
+
}
|
|
245
|
+
else {
|
|
246
|
+
result += ' 0:';
|
|
247
|
+
}
|
|
248
|
+
if (elapsed >= 1000) {
|
|
249
|
+
const s = Math.floor(elapsed / 1000);
|
|
250
|
+
result += padStart(s, '0');
|
|
251
|
+
}
|
|
252
|
+
else {
|
|
253
|
+
result += '00';
|
|
254
|
+
}
|
|
255
|
+
return result;
|
|
256
|
+
}
|
|
226
257
|
function observeFile(instance) {
|
|
227
258
|
instance.on('file:delete', (value, options) => this.delete(value, !!options?.emptyDir));
|
|
228
259
|
instance.on('file:copy', value => this.add(value));
|
|
@@ -249,12 +280,6 @@ function observeFile(instance) {
|
|
|
249
280
|
}
|
|
250
281
|
});
|
|
251
282
|
}
|
|
252
|
-
function setCpuUsage() {
|
|
253
|
-
const usage = core_1.Host.initCpuUsage(this);
|
|
254
|
-
if ('startCPU' in this) {
|
|
255
|
-
this.startCPU = usage;
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
283
|
function collectErrors() {
|
|
259
284
|
const result = [];
|
|
260
285
|
const clearModule = (instance) => {
|
|
@@ -284,10 +309,58 @@ function collectErrors() {
|
|
|
284
309
|
function rejectModule(err, type, hint) {
|
|
285
310
|
this.writeFail(["Handled rejection", this.moduleName + (hint ? ': ' + hint : '')], err, type);
|
|
286
311
|
}
|
|
312
|
+
const padStart = (value, char) => value > 9 ? value.toString() : char + value;
|
|
287
313
|
const checksumFile = (algorithm) => "checksum" + '.' + ((0, types_1.isString)(algorithm) ? algorithm.toLowerCase() : "sha256");
|
|
288
314
|
const checksumError = (algorithm) => new Error("Invalid parameters" + ` (${algorithm || "sha256"})`);
|
|
289
315
|
const isFunction = (value) => typeof value === 'function';
|
|
290
316
|
const ignoreAsset = (item, exists) => item.invalid || (0, types_1.hasBit)(item.flags, 1 | (!exists ? 128 : 0));
|
|
317
|
+
class Scheduler {
|
|
318
|
+
constructor() {
|
|
319
|
+
this.id = 0;
|
|
320
|
+
this.count = 0;
|
|
321
|
+
this.length = 0;
|
|
322
|
+
this.updated = 0;
|
|
323
|
+
this.titleWidth = 6;
|
|
324
|
+
this.valueWidth = 71;
|
|
325
|
+
this.limit = PROCESS_SUB_LIMIT;
|
|
326
|
+
this.queue = [];
|
|
327
|
+
this.transform = [];
|
|
328
|
+
this.status = [];
|
|
329
|
+
this.logCurrent = null;
|
|
330
|
+
if (LOG_FORMAT) {
|
|
331
|
+
this.titleWidth = LOG_FORMAT.title.width;
|
|
332
|
+
this.valueWidth = LOG_FORMAT.value.width;
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
add(id, url) {
|
|
336
|
+
const valueWidth = this.valueWidth;
|
|
337
|
+
if (url.length > valueWidth) {
|
|
338
|
+
url = url.substring(0, valueWidth - 3) + '...';
|
|
339
|
+
}
|
|
340
|
+
else {
|
|
341
|
+
url = url.padEnd(valueWidth);
|
|
342
|
+
}
|
|
343
|
+
this.status.push(new TaskStatus(id, url));
|
|
344
|
+
}
|
|
345
|
+
reset() {
|
|
346
|
+
this.status = [];
|
|
347
|
+
this.length = 0;
|
|
348
|
+
this.logCurrent = null;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
class TaskStatus {
|
|
352
|
+
constructor(id, url) {
|
|
353
|
+
this.id = id;
|
|
354
|
+
this.url = url;
|
|
355
|
+
this.receivedBytes = 0;
|
|
356
|
+
this.totalBytes = 0;
|
|
357
|
+
this.previousBytes = 0;
|
|
358
|
+
this.startTime = Date.now();
|
|
359
|
+
this.endTime = 0;
|
|
360
|
+
this.dataTime = null;
|
|
361
|
+
this.output = '';
|
|
362
|
+
}
|
|
363
|
+
}
|
|
291
364
|
class HttpDiskCache {
|
|
292
365
|
constructor(host, enabled) {
|
|
293
366
|
this.host = host;
|
|
@@ -318,9 +391,8 @@ class HttpDiskCache {
|
|
|
318
391
|
return true;
|
|
319
392
|
}
|
|
320
393
|
if (flags & 10) {
|
|
321
|
-
|
|
394
|
+
const ext = path.extname(uri.pathname).substring(1);
|
|
322
395
|
if (ext) {
|
|
323
|
-
ext = ext.substring(1);
|
|
324
396
|
if (flags & 2) {
|
|
325
397
|
if (!this._includeExt.some(value => value === ext)) {
|
|
326
398
|
return false;
|
|
@@ -333,7 +405,7 @@ class HttpDiskCache {
|
|
|
333
405
|
}
|
|
334
406
|
return flags & 1 ? this._include.includes(uri.origin) : !this._exclude.includes(uri.origin);
|
|
335
407
|
}
|
|
336
|
-
add(uri, etag, target, { buffer, contentLength = (0,
|
|
408
|
+
add(uri, etag, target, { buffer, contentLength = (0, util_1.getSize)(target) } = {}) {
|
|
337
409
|
let tempDir;
|
|
338
410
|
if (contentLength <= this.limit && (tempDir = this.host.getCacheDir(uri))) {
|
|
339
411
|
const baseDir = path.join(tempDir, etag);
|
|
@@ -570,7 +642,13 @@ class FileManager extends core_1.Host {
|
|
|
570
642
|
if (!super.loadSettings(settings, permission, password)) {
|
|
571
643
|
return false;
|
|
572
644
|
}
|
|
573
|
-
const { download, request, error, logger } = settings;
|
|
645
|
+
const { process: proc, download, request, error, logger } = settings;
|
|
646
|
+
if (proc && (0, types_1.isPlainObject)(proc.thread)) {
|
|
647
|
+
const limit = (0, util_1.asInt)(proc.thread.sub_limit);
|
|
648
|
+
if (limit > 0) {
|
|
649
|
+
PROCESS_SUB_LIMIT = limit;
|
|
650
|
+
}
|
|
651
|
+
}
|
|
574
652
|
if ((0, types_1.isPlainObject)(request)) {
|
|
575
653
|
let { timeout, disk, buffer, connect } = request;
|
|
576
654
|
if ((timeout = (0, util_1.fromSeconds)(timeout)) >= 0) {
|
|
@@ -585,23 +663,24 @@ class FileManager extends core_1.Host {
|
|
|
585
663
|
if (connect) {
|
|
586
664
|
this.defineHttpConnect(connect);
|
|
587
665
|
}
|
|
588
|
-
request_1.
|
|
666
|
+
request_1.loadSettings({ process: settings.process, request, download }, password);
|
|
589
667
|
}
|
|
590
668
|
if (error) {
|
|
591
|
-
const
|
|
592
|
-
if (
|
|
593
|
-
RECURSION_LIMIT =
|
|
669
|
+
const limit = (0, util_1.asInt)(error.recursion_limit);
|
|
670
|
+
if (limit >= 0 && limit < Infinity) {
|
|
671
|
+
RECURSION_LIMIT = limit;
|
|
594
672
|
}
|
|
595
673
|
}
|
|
596
674
|
if (logger) {
|
|
597
|
-
let
|
|
598
|
-
if (
|
|
675
|
+
let id = logger.session_id;
|
|
676
|
+
if (id === true) {
|
|
599
677
|
SESSION_LIMIT = 1000;
|
|
600
678
|
}
|
|
601
|
-
else if (
|
|
602
|
-
SESSION_LIMIT = Math.pow(10,
|
|
679
|
+
else if (id && (id = +id) > 0) {
|
|
680
|
+
SESSION_LIMIT = Math.pow(10, id);
|
|
603
681
|
}
|
|
604
682
|
}
|
|
683
|
+
LOG_FORMAT = this.LOG_FORMAT;
|
|
605
684
|
LOG_TIMEELAPSED = this.hasLogType(128);
|
|
606
685
|
LOG_TIMEPROCESS = this.hasLogType(256);
|
|
607
686
|
return true;
|
|
@@ -624,7 +703,7 @@ class FileManager extends core_1.Host {
|
|
|
624
703
|
item.flags &= ~(128 | 16);
|
|
625
704
|
break;
|
|
626
705
|
case 'watch':
|
|
627
|
-
if ((0, types_1.
|
|
706
|
+
if ((0, types_1.isObject)(item.watch) && item.watch.assets) {
|
|
628
707
|
delete item.watch.assets;
|
|
629
708
|
}
|
|
630
709
|
case 'buffer':
|
|
@@ -640,7 +719,7 @@ class FileManager extends core_1.Host {
|
|
|
640
719
|
case 'contentLength':
|
|
641
720
|
case 'invalid':
|
|
642
721
|
if (!exclusions.includes(attr)) {
|
|
643
|
-
|
|
722
|
+
item[attr] = undefined;
|
|
644
723
|
}
|
|
645
724
|
break;
|
|
646
725
|
}
|
|
@@ -722,7 +801,7 @@ class FileManager extends core_1.Host {
|
|
|
722
801
|
break;
|
|
723
802
|
}
|
|
724
803
|
}
|
|
725
|
-
let fail = [], missing = [];
|
|
804
|
+
let pass = 0, fail = [], missing = [];
|
|
726
805
|
try {
|
|
727
806
|
const filename = path.basename(from);
|
|
728
807
|
const files = [];
|
|
@@ -743,8 +822,11 @@ class FileManager extends core_1.Host {
|
|
|
743
822
|
if (hash !== previous) {
|
|
744
823
|
fail.push(pathname);
|
|
745
824
|
}
|
|
746
|
-
else
|
|
747
|
-
|
|
825
|
+
else {
|
|
826
|
+
if (verbose) {
|
|
827
|
+
process.stdout.write("+" + ' ' + pathname + '\n');
|
|
828
|
+
}
|
|
829
|
+
++pass;
|
|
748
830
|
}
|
|
749
831
|
valid = true;
|
|
750
832
|
}
|
|
@@ -760,13 +842,14 @@ class FileManager extends core_1.Host {
|
|
|
760
842
|
await Promise.all(tasks).then(group => {
|
|
761
843
|
for (const item of group) {
|
|
762
844
|
if (item) {
|
|
763
|
-
const [f, m] = item;
|
|
845
|
+
const [f, m, p] = item;
|
|
764
846
|
if (f.length) {
|
|
765
847
|
fail.push(...f);
|
|
766
848
|
}
|
|
767
849
|
if (m.length) {
|
|
768
850
|
missing.push(...m);
|
|
769
851
|
}
|
|
852
|
+
pass += p;
|
|
770
853
|
valid = true;
|
|
771
854
|
}
|
|
772
855
|
}
|
|
@@ -804,7 +887,7 @@ class FileManager extends core_1.Host {
|
|
|
804
887
|
writeLog(fail, "-");
|
|
805
888
|
writeLog(missing, "?");
|
|
806
889
|
}
|
|
807
|
-
return [fail, missing];
|
|
890
|
+
return [fail, missing, pass];
|
|
808
891
|
}
|
|
809
892
|
static createFileThread(host, file) {
|
|
810
893
|
return new FileThread(host, file, 0);
|
|
@@ -889,7 +972,6 @@ class FileManager extends core_1.Host {
|
|
|
889
972
|
}
|
|
890
973
|
constructor(baseDirectory, config, permission, postFinalize) {
|
|
891
974
|
super(config);
|
|
892
|
-
this.finalizeState = 0;
|
|
893
975
|
this.processTimeout = PROCESS_TIMEOUT.filemanager;
|
|
894
976
|
this.Document = [];
|
|
895
977
|
this.Task = [];
|
|
@@ -916,23 +998,25 @@ class FileManager extends core_1.Host {
|
|
|
916
998
|
this[_c] = false;
|
|
917
999
|
this[_d] = 0;
|
|
918
1000
|
this[_e] = false;
|
|
919
|
-
this[_f] =
|
|
920
|
-
this[_g] =
|
|
921
|
-
this[_h] =
|
|
922
|
-
this[_j] =
|
|
1001
|
+
this[_f] = 0;
|
|
1002
|
+
this[_g] = null;
|
|
1003
|
+
this[_h] = RECURSION_LIMIT;
|
|
1004
|
+
this[_j] = new Scheduler();
|
|
923
1005
|
this[_k] = {};
|
|
1006
|
+
this[_l] = [[0, 0], [0, 0], [0, 0]];
|
|
1007
|
+
this[_m] = {};
|
|
924
1008
|
if (isFunction(permission)) {
|
|
925
1009
|
postFinalize = permission;
|
|
926
1010
|
permission = undefined;
|
|
927
1011
|
}
|
|
928
1012
|
if (LOG_TIMEPROCESS) {
|
|
929
|
-
|
|
1013
|
+
core_1.Host.initCpuUsage(this);
|
|
930
1014
|
}
|
|
931
1015
|
const index = baseDirectory.length - 1;
|
|
932
1016
|
this[kBaseDirectory] = path.normalize(baseDirectory[index] === path.sep ? baseDirectory.substring(0, index) : baseDirectory);
|
|
933
1017
|
this.permission = permission && core_1.Host.isPermission(permission) ? permission : core_1.Host.getPermissionFromSettings();
|
|
934
1018
|
this.sessionId = (++SESSION_ID === SESSION_LIMIT ? SESSION_ID = 1 : SESSION_ID).toString();
|
|
935
|
-
const { assets = [], incremental, dataSource, timeout } = config;
|
|
1019
|
+
const { assets = [], incremental, dataSource, timeout, threads } = config;
|
|
936
1020
|
let targeted;
|
|
937
1021
|
for (let i = 0, length = assets.length; i < length; ++i) {
|
|
938
1022
|
const item = assets[i];
|
|
@@ -944,7 +1028,7 @@ class FileManager extends core_1.Host {
|
|
|
944
1028
|
this.taskAssets.push(item);
|
|
945
1029
|
}
|
|
946
1030
|
if (encoding && encoding !== 'utf8' && encoding !== 'utf-8') {
|
|
947
|
-
item.encoding = encoding
|
|
1031
|
+
item.encoding = encoding.endsWith('be') ? undefined : (0, types_1.getEncoding)(encoding);
|
|
948
1032
|
}
|
|
949
1033
|
if ((0, types_1.usingFlag)(item.flags)) {
|
|
950
1034
|
(targeted || (targeted = [])).push(item);
|
|
@@ -976,7 +1060,10 @@ class FileManager extends core_1.Host {
|
|
|
976
1060
|
else if (MEMORY.EXCLUDE.length) {
|
|
977
1061
|
this.cacheToMemory.exclude = MEMORY.EXCLUDE;
|
|
978
1062
|
}
|
|
979
|
-
|
|
1063
|
+
if (threads) {
|
|
1064
|
+
this.setTaskLimit(threads);
|
|
1065
|
+
}
|
|
1066
|
+
const request = new request_1();
|
|
980
1067
|
request.host = this;
|
|
981
1068
|
this.Request = request;
|
|
982
1069
|
if (this.aborted) {
|
|
@@ -989,7 +1076,7 @@ class FileManager extends core_1.Host {
|
|
|
989
1076
|
}
|
|
990
1077
|
}
|
|
991
1078
|
}
|
|
992
|
-
*[(_b = Symbol.toStringTag, _c = kRestarting, _d = kDelayed, _e = kCleared, _f =
|
|
1079
|
+
*[(_b = Symbol.toStringTag, _c = kRestarting, _d = kDelayed, _e = kCleared, _f = kFinalizedState, _g = KTimerMain, _h = kRecursionLimit, _j = kScheduler, _k = kProcessTimeout, _l = kDownloadStats, _m = kReplaceMap, Symbol.iterator)]() {
|
|
993
1080
|
for (const file of this.files) {
|
|
994
1081
|
yield file;
|
|
995
1082
|
}
|
|
@@ -1113,21 +1200,21 @@ class FileManager extends core_1.Host {
|
|
|
1113
1200
|
}
|
|
1114
1201
|
switch (name) {
|
|
1115
1202
|
case 'cloud': {
|
|
1116
|
-
const instance = new cloud_1
|
|
1203
|
+
const instance = new cloud_1(args[0]);
|
|
1117
1204
|
instance.host = this;
|
|
1118
1205
|
instance.init(this.config);
|
|
1119
1206
|
observeFile.call(this, instance);
|
|
1120
1207
|
return this.Cloud = instance;
|
|
1121
1208
|
}
|
|
1122
1209
|
case 'watch': {
|
|
1123
|
-
const instance = new watch_1
|
|
1210
|
+
const instance = new watch_1(...args);
|
|
1124
1211
|
instance.host = this;
|
|
1125
1212
|
instance.init(this.config);
|
|
1126
1213
|
observeFile.call(this, instance);
|
|
1127
1214
|
return this.Watch = instance;
|
|
1128
1215
|
}
|
|
1129
1216
|
case 'compress': {
|
|
1130
|
-
const instance = new compress_1
|
|
1217
|
+
const instance = new compress_1(args[0]);
|
|
1131
1218
|
instance.host = this;
|
|
1132
1219
|
instance.init();
|
|
1133
1220
|
observeFile.call(this, instance);
|
|
@@ -1160,13 +1247,14 @@ class FileManager extends core_1.Host {
|
|
|
1160
1247
|
clearAssets.call(this);
|
|
1161
1248
|
this.cleared = false;
|
|
1162
1249
|
this[kDelayed] = 0;
|
|
1250
|
+
this[kScheduler] = new Scheduler();
|
|
1163
1251
|
this.finalizeState = 0;
|
|
1164
1252
|
return true;
|
|
1165
1253
|
}
|
|
1166
1254
|
deleteFile(value, options = {}, callback) {
|
|
1167
1255
|
if (super.deleteFile(value, options, callback)) {
|
|
1168
1256
|
let emptyDir = false;
|
|
1169
|
-
if ((0, types_1.
|
|
1257
|
+
if ((0, types_1.isObject)(options)) {
|
|
1170
1258
|
if (options.emptyDir) {
|
|
1171
1259
|
emptyDir = true;
|
|
1172
1260
|
}
|
|
@@ -1197,10 +1285,18 @@ class FileManager extends core_1.Host {
|
|
|
1197
1285
|
if (this.aborted) {
|
|
1198
1286
|
return;
|
|
1199
1287
|
}
|
|
1200
|
-
|
|
1288
|
+
let target = args.shift();
|
|
1201
1289
|
switch (name) {
|
|
1202
1290
|
case 'document':
|
|
1203
|
-
if (
|
|
1291
|
+
if ((0, types_1.isString)(target)) {
|
|
1292
|
+
try {
|
|
1293
|
+
target = require(target);
|
|
1294
|
+
}
|
|
1295
|
+
catch (err) {
|
|
1296
|
+
this.checkPackage(err, target, ["Unable to load handler", this.moduleName], 4);
|
|
1297
|
+
}
|
|
1298
|
+
}
|
|
1299
|
+
if (isFunction(target) && target.prototype instanceof document_1) {
|
|
1204
1300
|
const instance = new target(...args);
|
|
1205
1301
|
instance.host = this;
|
|
1206
1302
|
instance.init(this.getDocumentAssets(instance), this.config);
|
|
@@ -1210,7 +1306,15 @@ class FileManager extends core_1.Host {
|
|
|
1210
1306
|
}
|
|
1211
1307
|
break;
|
|
1212
1308
|
case 'task':
|
|
1213
|
-
if (
|
|
1309
|
+
if ((0, types_1.isString)(target)) {
|
|
1310
|
+
try {
|
|
1311
|
+
target = require(target);
|
|
1312
|
+
}
|
|
1313
|
+
catch (err) {
|
|
1314
|
+
this.checkPackage(err, target, ["Unable to load handler", this.moduleName], 4);
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
1317
|
+
if (isFunction(target) && target.prototype instanceof task_1 && (0, types_1.isPlainObject)(args[0])) {
|
|
1214
1318
|
const instance = new target(...args);
|
|
1215
1319
|
instance.host = this;
|
|
1216
1320
|
instance.init(this.config);
|
|
@@ -1222,11 +1326,11 @@ class FileManager extends core_1.Host {
|
|
|
1222
1326
|
case 'cloud': {
|
|
1223
1327
|
const database = this.dataSourceItems.filter(item => item.source === 'cloud');
|
|
1224
1328
|
let instance;
|
|
1225
|
-
if (
|
|
1329
|
+
if ((0, types_1.isString)(target)) {
|
|
1226
1330
|
const module = args.shift();
|
|
1227
1331
|
if ((0, types_1.isObject)(module)) {
|
|
1228
1332
|
if (target === "@e-mc/cloud") {
|
|
1229
|
-
instance = new cloud_1
|
|
1333
|
+
instance = new cloud_1(module, database);
|
|
1230
1334
|
}
|
|
1231
1335
|
else {
|
|
1232
1336
|
try {
|
|
@@ -1245,7 +1349,7 @@ class FileManager extends core_1.Host {
|
|
|
1245
1349
|
}
|
|
1246
1350
|
}
|
|
1247
1351
|
else if ((0, types_1.isObject)(target)) {
|
|
1248
|
-
instance = new cloud_1
|
|
1352
|
+
instance = new cloud_1(target, database);
|
|
1249
1353
|
}
|
|
1250
1354
|
if (instance) {
|
|
1251
1355
|
instance.host = this;
|
|
@@ -1256,7 +1360,7 @@ class FileManager extends core_1.Host {
|
|
|
1256
1360
|
break;
|
|
1257
1361
|
}
|
|
1258
1362
|
case 'watch': {
|
|
1259
|
-
const instance = (0, types_1.isObject)(target) ? new watch_1
|
|
1363
|
+
const instance = (0, types_1.isObject)(target) ? new watch_1(target) : new watch_1({ interval: target, port: args[0], secure: { port: args[1] }, extensions: args[2] });
|
|
1260
1364
|
instance.host = this;
|
|
1261
1365
|
instance.init(this.config);
|
|
1262
1366
|
instance.whenModified = (assets, sanitize, postFinalize) => {
|
|
@@ -1287,12 +1391,21 @@ class FileManager extends core_1.Host {
|
|
|
1287
1391
|
let params = [];
|
|
1288
1392
|
for (const [mimeType, handler] of this.Image) {
|
|
1289
1393
|
const { constructor, params: trailing } = handler;
|
|
1290
|
-
if (
|
|
1394
|
+
if (mimeType === 'handler') {
|
|
1395
|
+
if (this.Image.size === 1) {
|
|
1396
|
+
manager.install('image', constructor, ...trailing);
|
|
1397
|
+
break;
|
|
1398
|
+
}
|
|
1399
|
+
params = trailing;
|
|
1400
|
+
}
|
|
1401
|
+
else if (params.length === 0) {
|
|
1291
1402
|
params = trailing;
|
|
1292
1403
|
}
|
|
1293
1404
|
mimeMap.set(mimeType, constructor);
|
|
1294
1405
|
}
|
|
1295
|
-
|
|
1406
|
+
if (mimeMap.size) {
|
|
1407
|
+
manager.install('image', mimeMap, ...params);
|
|
1408
|
+
}
|
|
1296
1409
|
}
|
|
1297
1410
|
if (this.Compress) {
|
|
1298
1411
|
manager.install('compress', this.Compress.module);
|
|
@@ -1313,25 +1426,43 @@ class FileManager extends core_1.Host {
|
|
|
1313
1426
|
};
|
|
1314
1427
|
return this.Watch = instance;
|
|
1315
1428
|
}
|
|
1316
|
-
case 'image':
|
|
1429
|
+
case 'image': {
|
|
1430
|
+
const createInstance = (mimeMap, mimeType, constructor, ...params) => {
|
|
1431
|
+
const instance = new constructor(...params);
|
|
1432
|
+
instance.host = this;
|
|
1433
|
+
instance.init(this.config);
|
|
1434
|
+
observeFile.call(this, instance);
|
|
1435
|
+
mimeMap.set(mimeType, { constructor, instance, params });
|
|
1436
|
+
return instance;
|
|
1437
|
+
};
|
|
1317
1438
|
if (target instanceof Map) {
|
|
1318
|
-
const
|
|
1439
|
+
const mimeMap = new Map();
|
|
1319
1440
|
for (const [mimeType, constructor] of target) {
|
|
1320
|
-
if (
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1441
|
+
if (constructor.prototype instanceof image_1) {
|
|
1442
|
+
createInstance(mimeMap, mimeType, constructor, args[0]);
|
|
1443
|
+
}
|
|
1444
|
+
}
|
|
1445
|
+
if (mimeMap.size) {
|
|
1446
|
+
this.Image = mimeMap;
|
|
1447
|
+
}
|
|
1448
|
+
}
|
|
1449
|
+
else {
|
|
1450
|
+
if ((0, types_1.isString)(target)) {
|
|
1451
|
+
try {
|
|
1452
|
+
target = require(target);
|
|
1453
|
+
}
|
|
1454
|
+
catch (err) {
|
|
1455
|
+
this.checkPackage(err, target, ["Unable to load handler", this.moduleName], 2048);
|
|
1326
1456
|
}
|
|
1327
1457
|
}
|
|
1328
|
-
if (
|
|
1329
|
-
this.Image = handler;
|
|
1458
|
+
if (isFunction(target) && target.prototype instanceof image_1) {
|
|
1459
|
+
return createInstance(this.Image ?? (this.Image = new Map()), 'handler', target, ...args);
|
|
1330
1460
|
}
|
|
1331
1461
|
}
|
|
1332
1462
|
break;
|
|
1463
|
+
}
|
|
1333
1464
|
case 'compress': {
|
|
1334
|
-
const instance = new compress_1
|
|
1465
|
+
const instance = new compress_1(target);
|
|
1335
1466
|
instance.host = this;
|
|
1336
1467
|
instance.init();
|
|
1337
1468
|
observeFile.call(this, instance);
|
|
@@ -1455,7 +1586,7 @@ class FileManager extends core_1.Host {
|
|
|
1455
1586
|
}
|
|
1456
1587
|
replace(file, replaceWith, mimeType) {
|
|
1457
1588
|
let ignoreExt, rewritePath;
|
|
1458
|
-
if ((0, types_1.
|
|
1589
|
+
if ((0, types_1.isObject)(mimeType)) {
|
|
1459
1590
|
({ mimeType, ignoreExt, rewritePath } = mimeType);
|
|
1460
1591
|
}
|
|
1461
1592
|
const { pathname, filename, localUri } = file;
|
|
@@ -1501,8 +1632,7 @@ class FileManager extends core_1.Host {
|
|
|
1501
1632
|
this[kReplaceMap][core_1.Host.joinPath(file.pathname, file.filename)] = file;
|
|
1502
1633
|
}
|
|
1503
1634
|
}
|
|
1504
|
-
|
|
1505
|
-
delete file.sourceUTF8;
|
|
1635
|
+
unsetContent(file);
|
|
1506
1636
|
return true;
|
|
1507
1637
|
}
|
|
1508
1638
|
rename(file, value) {
|
|
@@ -1602,18 +1732,18 @@ class FileManager extends core_1.Host {
|
|
|
1602
1732
|
}
|
|
1603
1733
|
resetState();
|
|
1604
1734
|
const log = this.config.log;
|
|
1605
|
-
let useNumeric
|
|
1606
|
-
if ((0, types_1.
|
|
1607
|
-
({ useNumeric, showSize
|
|
1735
|
+
let useNumeric, showSize;
|
|
1736
|
+
if ((0, types_1.isObject)(log)) {
|
|
1737
|
+
({ useNumeric, showSize } = log);
|
|
1608
1738
|
}
|
|
1609
|
-
const items =
|
|
1610
|
-
const size = (0,
|
|
1739
|
+
const items = showSize === false ? files : files.map(name => {
|
|
1740
|
+
const size = (0, util_1.getSize)(path.join(this.baseDirectory, name));
|
|
1611
1741
|
return { name, size: useNumeric ? size : (0, types_1.formatSize)(size) };
|
|
1612
1742
|
});
|
|
1613
1743
|
this.emit('end', items, errors, this.collectLog());
|
|
1614
1744
|
this.done = true;
|
|
1615
1745
|
})
|
|
1616
|
-
.catch(err => aborted(err));
|
|
1746
|
+
.catch((err) => aborted(err));
|
|
1617
1747
|
}
|
|
1618
1748
|
}
|
|
1619
1749
|
hasDocument(instance, document) {
|
|
@@ -1638,7 +1768,7 @@ class FileManager extends core_1.Host {
|
|
|
1638
1768
|
}
|
|
1639
1769
|
else {
|
|
1640
1770
|
try {
|
|
1641
|
-
const errorAbsolute = (value) => (0, types_1.errorValue)(
|
|
1771
|
+
const errorAbsolute = (value) => (0, types_1.errorValue)("Path is not absolute", value);
|
|
1642
1772
|
if (!(file.uri = core_1.Host.resolveFile(uri))) {
|
|
1643
1773
|
throw errorAbsolute(uri);
|
|
1644
1774
|
}
|
|
@@ -1713,7 +1843,7 @@ class FileManager extends core_1.Host {
|
|
|
1713
1843
|
file.encoding = (0, types_1.getEncoding)(file.encoding);
|
|
1714
1844
|
if (file.buffer) {
|
|
1715
1845
|
sourceUTF8 = file.buffer.toString(file.encoding);
|
|
1716
|
-
|
|
1846
|
+
file.buffer = null;
|
|
1717
1847
|
}
|
|
1718
1848
|
else if (uri || (uri = file.localUri)) {
|
|
1719
1849
|
try {
|
|
@@ -1734,19 +1864,22 @@ class FileManager extends core_1.Host {
|
|
|
1734
1864
|
if (file.buffer) {
|
|
1735
1865
|
return (typeof minStreamSize === 'number' ? Promise.resolve(file.buffer) : file.buffer);
|
|
1736
1866
|
}
|
|
1737
|
-
|
|
1738
|
-
if (
|
|
1867
|
+
let { base64, localUri } = file;
|
|
1868
|
+
if (localUri) {
|
|
1739
1869
|
if (typeof minStreamSize === 'number') {
|
|
1740
|
-
return core_1.Host.streamFile(
|
|
1870
|
+
return core_1.Host.streamFile(localUri, { minStreamSize, cache: false, signal: this.signal });
|
|
1741
1871
|
}
|
|
1742
1872
|
try {
|
|
1743
|
-
return fs.readFileSync(
|
|
1873
|
+
return fs.readFileSync(localUri);
|
|
1744
1874
|
}
|
|
1745
1875
|
catch (err) {
|
|
1746
|
-
|
|
1876
|
+
if (!(base64 && core_1.Host.isErrorCode(err, 'ENOENT'))) {
|
|
1877
|
+
this.writeFail(["Unable to read file", path.basename(localUri)], err, 32);
|
|
1878
|
+
base64 = undefined;
|
|
1879
|
+
}
|
|
1747
1880
|
}
|
|
1748
1881
|
}
|
|
1749
|
-
const result =
|
|
1882
|
+
const result = base64 ? Buffer.from(base64, 'base64') : null;
|
|
1750
1883
|
return (typeof minStreamSize === 'number' ? Promise.resolve(result) : result);
|
|
1751
1884
|
}
|
|
1752
1885
|
getCacheDir(url, createDir = true) {
|
|
@@ -1943,20 +2076,20 @@ class FileManager extends core_1.Host {
|
|
|
1943
2076
|
const instance = this.Compress;
|
|
1944
2077
|
if (instance && compress && localUri && (this.has(localUri) || (0, types_1.existsFlag)(file.flags))) {
|
|
1945
2078
|
const tasks = [];
|
|
1946
|
-
for (const
|
|
1947
|
-
const { format, condition } =
|
|
2079
|
+
for (const options of compress) {
|
|
2080
|
+
const { format, condition } = options;
|
|
1948
2081
|
let output;
|
|
1949
2082
|
switch (format) {
|
|
1950
2083
|
case 'br':
|
|
1951
|
-
|
|
2084
|
+
options.mimeType || (options.mimeType = file.mimeType);
|
|
1952
2085
|
case 'gz':
|
|
1953
2086
|
output = localUri + '.' + format;
|
|
1954
2087
|
break;
|
|
1955
2088
|
case 'woff':
|
|
1956
2089
|
case 'woff2':
|
|
1957
2090
|
output = (0, types_1.renameExt)(localUri, format);
|
|
1958
|
-
|
|
1959
|
-
|
|
2091
|
+
options.filename = file.filename;
|
|
2092
|
+
options.etag = file.etag;
|
|
1960
2093
|
break;
|
|
1961
2094
|
default:
|
|
1962
2095
|
if (format && typeof instance.compressors[format] === 'function') {
|
|
@@ -1967,19 +2100,20 @@ class FileManager extends core_1.Host {
|
|
|
1967
2100
|
if (output && (!condition || withinSizeRange(localUri, condition))) {
|
|
1968
2101
|
try {
|
|
1969
2102
|
if (overwrite || !fs.existsSync(output) || !(0, types_1.existsFlag)(file.flags) && fs.statSync(output).mtimeMs < this.startTime) {
|
|
1970
|
-
|
|
1971
|
-
tasks.push(instance.tryFile(file.buffer || localUri, output,
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
if (condition?.includes('%') && (0, lib_v4_1.getSize)(result) >= (0, lib_v4_1.getSize)(localUri)) {
|
|
1977
|
-
queueMicrotask(() => this.deleteFile(result));
|
|
2103
|
+
options.timeout = this[kProcessTimeout].compress ?? PROCESS_TIMEOUT.compress;
|
|
2104
|
+
tasks.push(instance.tryFile(file.buffer || localUri, output, options).then(() => {
|
|
2105
|
+
const outFile = options.outFile;
|
|
2106
|
+
if (outFile) {
|
|
2107
|
+
if (condition?.includes('%') && (0, util_1.getSize)(outFile) >= (0, util_1.getSize)(localUri)) {
|
|
2108
|
+
queueMicrotask(() => this.deleteFile(outFile));
|
|
1978
2109
|
}
|
|
1979
2110
|
else {
|
|
1980
|
-
this.add(
|
|
2111
|
+
this.add(outFile, file, types_1.FILE_TYPE.COMPRESSED);
|
|
1981
2112
|
}
|
|
1982
2113
|
}
|
|
2114
|
+
})
|
|
2115
|
+
.catch((err) => {
|
|
2116
|
+
this.writeFail(["Unable to compress file", path.basename(localUri)], err, { type: 8, startTime: options.startTime });
|
|
1983
2117
|
}));
|
|
1984
2118
|
}
|
|
1985
2119
|
}
|
|
@@ -1994,10 +2128,17 @@ class FileManager extends core_1.Host {
|
|
|
1994
2128
|
}
|
|
1995
2129
|
return [];
|
|
1996
2130
|
}
|
|
1997
|
-
async transformAsset(data, parent) {
|
|
2131
|
+
async transformAsset(data, parent, override) {
|
|
1998
2132
|
const file = data.file;
|
|
1999
2133
|
const localUri = this.getLocalUri(data);
|
|
2000
2134
|
if (!this.aborted) {
|
|
2135
|
+
if (!override) {
|
|
2136
|
+
const processTask = this[kScheduler];
|
|
2137
|
+
if (processTask.queue.length) {
|
|
2138
|
+
processTask.transform.push([data, parent]);
|
|
2139
|
+
return false;
|
|
2140
|
+
}
|
|
2141
|
+
}
|
|
2001
2142
|
this.processing.push(data);
|
|
2002
2143
|
let mimeType = file.mimeType;
|
|
2003
2144
|
if ((mimeType?.endsWith('unknown') || file.commands && !mimeType) && (mimeType = await this.findMime(data.file, true))) {
|
|
@@ -2007,62 +2148,44 @@ class FileManager extends core_1.Host {
|
|
|
2007
2148
|
const errorAsset = (instance, err, type = 4) => rejectModule.call(instance, err, type, path.basename(localUri));
|
|
2008
2149
|
if (file.tasks) {
|
|
2009
2150
|
const taskName = [];
|
|
2151
|
+
let handler;
|
|
2010
2152
|
for (const { task, handler: moduleName, preceding } of file.tasks) {
|
|
2011
|
-
if (task && preceding && moduleName && !taskName.includes(moduleName) &&
|
|
2012
|
-
const
|
|
2013
|
-
if (
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
(instance.using ? instance.using(data) : constructor.using.call(this, instance, [file], true))
|
|
2020
|
-
.then(() => this.closeThread(instance, data))
|
|
2021
|
-
.catch(err => {
|
|
2022
|
-
this.closeThread(instance, data);
|
|
2023
|
-
errorAsset(instance, err);
|
|
2024
|
-
});
|
|
2025
|
-
}
|
|
2026
|
-
}
|
|
2027
|
-
else if (!this.aborted) {
|
|
2028
|
-
this.addProcessTimeout(instance, file, this.getProcessTimeout(handler));
|
|
2029
|
-
await (instance.using ? instance.using(data) : constructor.using.call(this, instance, [file], true));
|
|
2030
|
-
}
|
|
2031
|
-
}
|
|
2032
|
-
catch (err) {
|
|
2033
|
-
errorAsset(instance, err);
|
|
2153
|
+
if (task && preceding && moduleName && !taskName.includes(moduleName) && (handler = this.Task.find(item => item.instance.moduleName === moduleName))) {
|
|
2154
|
+
const instance = handler.instance;
|
|
2155
|
+
if (instance.using && !data.aborted) {
|
|
2156
|
+
if (instance.threadable) {
|
|
2157
|
+
if (this.openThread(instance, data, this.getProcessTimeout(handler))) {
|
|
2158
|
+
instance.using(data)
|
|
2159
|
+
.finally(() => this.closeThread(instance, data))
|
|
2160
|
+
.catch((err) => errorAsset(instance, err));
|
|
2034
2161
|
}
|
|
2035
|
-
taskName.push(moduleName);
|
|
2036
2162
|
}
|
|
2163
|
+
else if (!this.aborted) {
|
|
2164
|
+
this.addProcessTimeout(instance, file, this.getProcessTimeout(handler));
|
|
2165
|
+
await instance.using(data).catch((err) => errorAsset(instance, err));
|
|
2166
|
+
}
|
|
2167
|
+
taskName.push(moduleName);
|
|
2037
2168
|
}
|
|
2038
2169
|
}
|
|
2039
2170
|
}
|
|
2040
2171
|
}
|
|
2041
|
-
if (image_1.
|
|
2172
|
+
if (image_1.isBinary(mimeType)) {
|
|
2042
2173
|
if (file.commands && this.Image) {
|
|
2043
2174
|
const handler = this.Image.get(mimeType) || this.Image.get('handler');
|
|
2044
2175
|
if (handler) {
|
|
2045
2176
|
const { instance, constructor } = handler;
|
|
2046
2177
|
for (const command of file.commands) {
|
|
2047
|
-
if (withinSizeRange(localUri, command, constructor.REGEXP_SIZERANGE) &&
|
|
2048
|
-
|
|
2049
|
-
if (instance.
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
.catch(err => {
|
|
2054
|
-
this.closeThread(instance, data);
|
|
2055
|
-
errorAsset(instance, err, 2048);
|
|
2056
|
-
});
|
|
2057
|
-
}
|
|
2058
|
-
}
|
|
2059
|
-
else if (!this.aborted) {
|
|
2060
|
-
this.addProcessTimeout(instance, file, this.getProcessTimeout(handler));
|
|
2061
|
-
await (instance.using ? instance.using(data, command) : constructor.using.call(this, instance, data, command));
|
|
2178
|
+
if (withinSizeRange(localUri, command, constructor.REGEXP_SIZERANGE) && instance.using && !data.aborted) {
|
|
2179
|
+
if (instance.threadable) {
|
|
2180
|
+
if (this.openThread(instance, data, this.getProcessTimeout(handler))) {
|
|
2181
|
+
instance.using(data, command)
|
|
2182
|
+
.finally(() => this.closeThread(instance, data))
|
|
2183
|
+
.catch((err) => errorAsset(instance, err, 2048));
|
|
2062
2184
|
}
|
|
2063
2185
|
}
|
|
2064
|
-
|
|
2065
|
-
|
|
2186
|
+
else if (!this.aborted) {
|
|
2187
|
+
this.addProcessTimeout(instance, file, this.getProcessTimeout(handler));
|
|
2188
|
+
await instance.using(data, command).catch((err) => errorAsset(instance, err, 2048));
|
|
2066
2189
|
}
|
|
2067
2190
|
}
|
|
2068
2191
|
}
|
|
@@ -2071,26 +2194,18 @@ class FileManager extends core_1.Host {
|
|
|
2071
2194
|
}
|
|
2072
2195
|
else if (file.document) {
|
|
2073
2196
|
for (const handler of this.Document) {
|
|
2074
|
-
const
|
|
2075
|
-
if (this.hasDocument(instance, file.document) &&
|
|
2076
|
-
|
|
2077
|
-
if (instance.
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
.catch(err => {
|
|
2082
|
-
this.closeThread(instance, data);
|
|
2083
|
-
errorAsset(instance, err);
|
|
2084
|
-
});
|
|
2085
|
-
}
|
|
2086
|
-
}
|
|
2087
|
-
else if (!this.aborted) {
|
|
2088
|
-
this.addProcessTimeout(instance, file, this.getProcessTimeout(handler));
|
|
2089
|
-
await (instance.using ? instance.using(data) : constructor.using.call(this, instance, file));
|
|
2197
|
+
const instance = handler.instance;
|
|
2198
|
+
if (this.hasDocument(instance, file.document) && instance.using && !data.aborted) {
|
|
2199
|
+
if (instance.threadable) {
|
|
2200
|
+
if (this.openThread(instance, data, this.getProcessTimeout(handler))) {
|
|
2201
|
+
instance.using(data)
|
|
2202
|
+
.finally(() => this.closeThread(instance, data))
|
|
2203
|
+
.catch((err) => errorAsset(instance, err));
|
|
2090
2204
|
}
|
|
2091
2205
|
}
|
|
2092
|
-
|
|
2093
|
-
|
|
2206
|
+
else if (!this.aborted) {
|
|
2207
|
+
this.addProcessTimeout(instance, file, this.getProcessTimeout(handler));
|
|
2208
|
+
await instance.using(data).catch((err) => errorAsset(instance, err));
|
|
2094
2209
|
}
|
|
2095
2210
|
}
|
|
2096
2211
|
}
|
|
@@ -2104,6 +2219,7 @@ class FileManager extends core_1.Host {
|
|
|
2104
2219
|
this.deleteFile(localUri, { emptyDir: true });
|
|
2105
2220
|
}
|
|
2106
2221
|
});
|
|
2222
|
+
return true;
|
|
2107
2223
|
}
|
|
2108
2224
|
openThread(instance, data, timeout = 0) {
|
|
2109
2225
|
if (this.aborted || data.aborted) {
|
|
@@ -2193,7 +2309,7 @@ class FileManager extends core_1.Host {
|
|
|
2193
2309
|
result = value;
|
|
2194
2310
|
}
|
|
2195
2311
|
else if (typeof value === 'string') {
|
|
2196
|
-
if ((result = (0,
|
|
2312
|
+
if ((result = (0, util_1.byteLength)(value, encoding)) === 0) {
|
|
2197
2313
|
return 0;
|
|
2198
2314
|
}
|
|
2199
2315
|
}
|
|
@@ -2456,14 +2572,14 @@ class FileManager extends core_1.Host {
|
|
|
2456
2572
|
}
|
|
2457
2573
|
};
|
|
2458
2574
|
const checkQueue = (file, localUri, pathname, content) => {
|
|
2459
|
-
var
|
|
2575
|
+
var _o;
|
|
2460
2576
|
if (!createFolder(file, pathname)) {
|
|
2461
2577
|
return true;
|
|
2462
2578
|
}
|
|
2463
2579
|
const { bundleId, bundleIndex = -1 } = file;
|
|
2464
2580
|
if (!(0, types_1.isEmpty)(bundleId) && bundleIndex >= 0) {
|
|
2465
2581
|
const items = appending[localUri] || (appending[localUri] = []);
|
|
2466
|
-
bundling[
|
|
2582
|
+
bundling[_o = file.uri] || (bundling[_o] = []);
|
|
2467
2583
|
if (bundleIndex > 0) {
|
|
2468
2584
|
items[bundleIndex - 1] = file;
|
|
2469
2585
|
let url, parent;
|
|
@@ -2575,7 +2691,7 @@ class FileManager extends core_1.Host {
|
|
|
2575
2691
|
value = value.toString(encoding);
|
|
2576
2692
|
}
|
|
2577
2693
|
const url = queue.url;
|
|
2578
|
-
if (etag &&
|
|
2694
|
+
if (etag && cacheToMemory.has(url)) {
|
|
2579
2695
|
cacheToMemory.add(url, encodeURIComponent(etag), value, {
|
|
2580
2696
|
encoding,
|
|
2581
2697
|
contentLength: queue.contentLength,
|
|
@@ -2624,7 +2740,7 @@ class FileManager extends core_1.Host {
|
|
|
2624
2740
|
const baseDir = path.join(this.getCacheDir(url), etagDir);
|
|
2625
2741
|
pipeTo = path.join(baseDir, path.basename(localUri));
|
|
2626
2742
|
try {
|
|
2627
|
-
if (valid && (0,
|
|
2743
|
+
if (valid && (0, util_1.hasSize)(pipeTo)) {
|
|
2628
2744
|
const buffer = fs.readFileSync(pipeTo, { encoding });
|
|
2629
2745
|
if (checksumValid(buffer)) {
|
|
2630
2746
|
verifyBundle(buffer, etag, true);
|
|
@@ -2651,38 +2767,40 @@ class FileManager extends core_1.Host {
|
|
|
2651
2767
|
return true;
|
|
2652
2768
|
};
|
|
2653
2769
|
}
|
|
2654
|
-
tasks.push(
|
|
2655
|
-
.
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2770
|
+
tasks.push(new Promise((resolve, reject) => {
|
|
2771
|
+
this.scheduleTask(url || uri, options, (data) => {
|
|
2772
|
+
if (data) {
|
|
2773
|
+
verifyBundle(data, etag);
|
|
2774
|
+
}
|
|
2775
|
+
else {
|
|
2776
|
+
queue.invalid = true;
|
|
2777
|
+
}
|
|
2778
|
+
const downloaded = bundling[uri];
|
|
2779
|
+
if ((0, types_1.isArray)(downloaded)) {
|
|
2780
|
+
if (data && !queue.invalid) {
|
|
2781
|
+
if (typeof data === 'string') {
|
|
2782
|
+
queue.sourceUTF8 = data;
|
|
2783
|
+
}
|
|
2784
|
+
else {
|
|
2785
|
+
queue.buffer = data;
|
|
2786
|
+
}
|
|
2787
|
+
copyDownload(queue, downloaded);
|
|
2667
2788
|
}
|
|
2668
2789
|
else {
|
|
2669
|
-
|
|
2790
|
+
downloaded.forEach(item => item.invalid = true);
|
|
2670
2791
|
}
|
|
2671
|
-
copyDownload(queue, downloaded);
|
|
2672
2792
|
}
|
|
2673
|
-
|
|
2674
|
-
|
|
2793
|
+
if (tempFile && (!pipeTo || core_1.Host.isPath(pipeTo) || !core_1.Host.renameFile(tempFile, pipeTo, false))) {
|
|
2794
|
+
queueMicrotask(() => fs.unlink(tempFile, () => { }));
|
|
2675
2795
|
}
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
queueMicrotask(() => fs.unlink(tempFile, () => { }));
|
|
2685
|
-
}
|
|
2796
|
+
resolve();
|
|
2797
|
+
}, (err) => {
|
|
2798
|
+
queue.invalid = true;
|
|
2799
|
+
if (tempFile) {
|
|
2800
|
+
queueMicrotask(() => fs.unlink(tempFile, () => { }));
|
|
2801
|
+
}
|
|
2802
|
+
reject(err);
|
|
2803
|
+
}, 1);
|
|
2686
2804
|
}));
|
|
2687
2805
|
}
|
|
2688
2806
|
else if (type) {
|
|
@@ -2693,19 +2811,21 @@ class FileManager extends core_1.Host {
|
|
|
2693
2811
|
tasks.push(Promise.reject(!pathname ? new Error("Unable to create temp directory") : (0, types_1.errorValue)("MIME not found", uri)));
|
|
2694
2812
|
break;
|
|
2695
2813
|
}
|
|
2696
|
-
tasks.push(
|
|
2697
|
-
.
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2814
|
+
tasks.push(new Promise((resolve, reject) => {
|
|
2815
|
+
this.scheduleTask(queue.url || uri, { pathname, binOpts: queue.binOpts || file.binOpts }, (result) => {
|
|
2816
|
+
if (result.length) {
|
|
2817
|
+
verifyBundle(bundleTorrent.call(this, result, mimeType, encoding));
|
|
2818
|
+
}
|
|
2819
|
+
else {
|
|
2820
|
+
queue.invalid = true;
|
|
2821
|
+
}
|
|
2822
|
+
queueMicrotask(() => core_1.Host.removeDir(pathname));
|
|
2823
|
+
resolve();
|
|
2824
|
+
}, (err) => {
|
|
2702
2825
|
queue.invalid = true;
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
.catch(() => {
|
|
2707
|
-
queue.invalid = true;
|
|
2708
|
-
queueMicrotask(() => core_1.Host.removeDir(pathname));
|
|
2826
|
+
queueMicrotask(() => core_1.Host.removeDir(pathname));
|
|
2827
|
+
reject(err);
|
|
2828
|
+
}, 4);
|
|
2709
2829
|
}));
|
|
2710
2830
|
}
|
|
2711
2831
|
else if (this.canRead(uri)) {
|
|
@@ -2727,7 +2847,7 @@ class FileManager extends core_1.Host {
|
|
|
2727
2847
|
if (tasks.length) {
|
|
2728
2848
|
success = await Promise.all(tasks)
|
|
2729
2849
|
.then(() => true)
|
|
2730
|
-
.catch(err => {
|
|
2850
|
+
.catch((err) => {
|
|
2731
2851
|
this.writeFail(["Unable to download file", 'bundle: ' + path.basename(localUri)], err, 1024);
|
|
2732
2852
|
return false;
|
|
2733
2853
|
});
|
|
@@ -2747,8 +2867,7 @@ class FileManager extends core_1.Host {
|
|
|
2747
2867
|
}
|
|
2748
2868
|
else {
|
|
2749
2869
|
file.invalid = true;
|
|
2750
|
-
|
|
2751
|
-
delete file.sourceUTF8;
|
|
2870
|
+
unsetContent(file);
|
|
2752
2871
|
this.completeAsyncTask(error);
|
|
2753
2872
|
queueMicrotask(() => this.deleteFile(localUri, { emptyDir: true }));
|
|
2754
2873
|
}
|
|
@@ -2797,7 +2916,7 @@ class FileManager extends core_1.Host {
|
|
|
2797
2916
|
this.writeFail(["Unable to download file", uri], err, err instanceof Error && err.message.startsWith("Timeout was exceeded") ? 16384 : 1024);
|
|
2798
2917
|
};
|
|
2799
2918
|
const errorPermission = (file) => {
|
|
2800
|
-
this.writeFail(["Unable to read file", file.uri], (0, types_1.errorValue)(
|
|
2919
|
+
this.writeFail(["Unable to read file", file.uri], (0, types_1.errorValue)("Operation not permitted", file.uri || "Unknown"), 8192);
|
|
2801
2920
|
file.invalid = true;
|
|
2802
2921
|
};
|
|
2803
2922
|
const createFolder = (file, pathname) => {
|
|
@@ -2810,7 +2929,7 @@ class FileManager extends core_1.Host {
|
|
|
2810
2929
|
}
|
|
2811
2930
|
else {
|
|
2812
2931
|
file.invalid = true;
|
|
2813
|
-
this.writeFail("Unable to create directory", (0, types_1.errorValue)(
|
|
2932
|
+
this.writeFail("Unable to create directory", (0, types_1.errorValue)("Path not found", pathname));
|
|
2814
2933
|
return false;
|
|
2815
2934
|
}
|
|
2816
2935
|
}
|
|
@@ -3030,26 +3149,40 @@ class FileManager extends core_1.Host {
|
|
|
3030
3149
|
}
|
|
3031
3150
|
const uri = item.uri;
|
|
3032
3151
|
if (!uri) {
|
|
3152
|
+
if (item.buffer) {
|
|
3153
|
+
if (createFolder(item, pathname)) {
|
|
3154
|
+
this.performAsyncTask();
|
|
3155
|
+
this.scheduleTask(localUri, item.buffer, () => {
|
|
3156
|
+
fileReceived(item, localUri, null, false, true);
|
|
3157
|
+
}, (err) => {
|
|
3158
|
+
fileReceived(item, localUri, err);
|
|
3159
|
+
}).then(code => {
|
|
3160
|
+
if (code === -1) {
|
|
3161
|
+
fileReceived(item, localUri, (0, types_1.errorValue)("Not able to read buffer", path.basename(localUri)));
|
|
3162
|
+
}
|
|
3163
|
+
});
|
|
3164
|
+
}
|
|
3165
|
+
continue;
|
|
3166
|
+
}
|
|
3033
3167
|
item.invalid = true;
|
|
3034
3168
|
continue;
|
|
3035
3169
|
}
|
|
3036
|
-
const checkDest = (src) => staging || !(0,
|
|
3170
|
+
const checkDest = (src) => staging || !(0, util_1.hasSameStat)(src, localUri);
|
|
3037
3171
|
const type = item.fetchType || 0;
|
|
3038
3172
|
const bundleMain = item.bundleIndex === 0 && !(0, types_1.isEmpty)(item.bundleId);
|
|
3039
|
-
|
|
3040
|
-
if (isHttp || type === 2) {
|
|
3173
|
+
if (type === 1 || type === 2) {
|
|
3041
3174
|
let checkEtag;
|
|
3042
3175
|
if (bundling[uri] && (0, types_1.isEmpty)(item.bundleId)) {
|
|
3043
3176
|
bundling[uri].push(item);
|
|
3044
3177
|
}
|
|
3045
3178
|
else if ((checkEtag = hasEtag(item)) || !checkQueue(item, localUri, pathname)) {
|
|
3046
|
-
if (
|
|
3047
|
-
fileReceived(item, localUri);
|
|
3179
|
+
if (type !== 1 && !checkDest(uri)) {
|
|
3180
|
+
fileReceived(item, localUri, null);
|
|
3048
3181
|
}
|
|
3049
3182
|
else if (!checkEtag && downloading[uri]) {
|
|
3050
3183
|
downloading[uri].push(item);
|
|
3051
3184
|
}
|
|
3052
|
-
else if (
|
|
3185
|
+
else if (type !== 1 && !this.canRead(uri)) {
|
|
3053
3186
|
errorPermission(item);
|
|
3054
3187
|
}
|
|
3055
3188
|
else {
|
|
@@ -3058,7 +3191,7 @@ class FileManager extends core_1.Host {
|
|
|
3058
3191
|
}
|
|
3059
3192
|
const url = item.url;
|
|
3060
3193
|
let cacheDir, cacheBuffer, mainEtag, encoding = item.encoding, client = null;
|
|
3061
|
-
if (
|
|
3194
|
+
if (type === 1) {
|
|
3062
3195
|
cacheDir = cacheToDisk.has(url) && isCacheable(item);
|
|
3063
3196
|
cacheBuffer = cacheToMemory.has(url);
|
|
3064
3197
|
mainEtag = (0, types_1.mainFlag)(item.flags) && (cacheEtag || item.incremental === "etag");
|
|
@@ -3073,7 +3206,7 @@ class FileManager extends core_1.Host {
|
|
|
3073
3206
|
request.encoding = encoding;
|
|
3074
3207
|
request.pipeTo = localUri;
|
|
3075
3208
|
request.statusMessage = location + (bundleMain ? ' (0)' : '');
|
|
3076
|
-
if (
|
|
3209
|
+
if (type === 1) {
|
|
3077
3210
|
request.method = 'GET';
|
|
3078
3211
|
request.httpVersion = (originCount[request.url.origin] || 0) <= 1 ? 1 : undefined;
|
|
3079
3212
|
request.connected = headers => {
|
|
@@ -3086,15 +3219,14 @@ class FileManager extends core_1.Host {
|
|
|
3086
3219
|
request.method = undefined;
|
|
3087
3220
|
request.httpVersion = 1;
|
|
3088
3221
|
}
|
|
3089
|
-
this.
|
|
3090
|
-
.then(data => {
|
|
3222
|
+
this.scheduleTask(request.url, request, (data) => {
|
|
3091
3223
|
if (typeof data === 'string') {
|
|
3092
3224
|
item.sourceUTF8 = data;
|
|
3093
3225
|
}
|
|
3094
3226
|
else if (data) {
|
|
3095
3227
|
item.buffer = data;
|
|
3096
3228
|
}
|
|
3097
|
-
fileReceived(item, localUri, null,
|
|
3229
|
+
fileReceived(item, localUri, null, type === 1);
|
|
3098
3230
|
if (etagDir) {
|
|
3099
3231
|
queueMicrotask(() => {
|
|
3100
3232
|
if (cacheBuffer && data) {
|
|
@@ -3105,8 +3237,9 @@ class FileManager extends core_1.Host {
|
|
|
3105
3237
|
}
|
|
3106
3238
|
});
|
|
3107
3239
|
}
|
|
3108
|
-
})
|
|
3109
|
-
|
|
3240
|
+
}, (err) => {
|
|
3241
|
+
errorRequest(item, err);
|
|
3242
|
+
}, 1);
|
|
3110
3243
|
};
|
|
3111
3244
|
this.performAsyncTask();
|
|
3112
3245
|
if (cacheDir || cacheBuffer || mainEtag) {
|
|
@@ -3122,7 +3255,7 @@ class FileManager extends core_1.Host {
|
|
|
3122
3255
|
const etag = applyHeaders(item, res.headers, false, mainEtag);
|
|
3123
3256
|
let tempDir, etagDir;
|
|
3124
3257
|
if (etag) {
|
|
3125
|
-
if (cacheDir) {
|
|
3258
|
+
if (cacheDir || mainEtag) {
|
|
3126
3259
|
tempDir = this.getCacheDir(href);
|
|
3127
3260
|
}
|
|
3128
3261
|
etagDir = encodeURIComponent(etag);
|
|
@@ -3161,7 +3294,7 @@ class FileManager extends core_1.Host {
|
|
|
3161
3294
|
}
|
|
3162
3295
|
};
|
|
3163
3296
|
let pipeAs;
|
|
3164
|
-
if (tempDir && (0,
|
|
3297
|
+
if (tempDir && (0, util_1.hasSize)(pipeAs = path.join(tempDir, etagDir, path.basename(localUri)))) {
|
|
3165
3298
|
if (!checkEtag) {
|
|
3166
3299
|
if (cacheBuffer && !buffer) {
|
|
3167
3300
|
buffer = fs.readFileSync(pipeAs, encoding);
|
|
@@ -3231,7 +3364,7 @@ class FileManager extends core_1.Host {
|
|
|
3231
3364
|
const location = res.headers.location;
|
|
3232
3365
|
if (location && ++redirects <= HTTP_CLIENT.redirectLimit) {
|
|
3233
3366
|
try {
|
|
3234
|
-
checkHeaders.call(this, new URL(request_1.
|
|
3367
|
+
checkHeaders.call(this, new URL(request_1.fromURL(href, location)));
|
|
3235
3368
|
return;
|
|
3236
3369
|
}
|
|
3237
3370
|
catch {
|
|
@@ -3281,10 +3414,9 @@ class FileManager extends core_1.Host {
|
|
|
3281
3414
|
}
|
|
3282
3415
|
downloading[uri] = [];
|
|
3283
3416
|
this.performAsyncTask();
|
|
3284
|
-
this.
|
|
3285
|
-
.then(result => {
|
|
3417
|
+
this.scheduleTask(item.url || uri, { pathname, binOpts: item.binOpts }, (result) => {
|
|
3286
3418
|
if (result.length === 0) {
|
|
3287
|
-
errorRequest(item, (0, types_1.errorValue)(
|
|
3419
|
+
errorRequest(item, (0, types_1.errorValue)("No files were successfully downloaded", uri));
|
|
3288
3420
|
}
|
|
3289
3421
|
else if (bundleMain) {
|
|
3290
3422
|
const encoding = item.encoding || (item.encoding = 'utf-8');
|
|
@@ -3330,8 +3462,9 @@ class FileManager extends core_1.Host {
|
|
|
3330
3462
|
this.completeAsyncTask();
|
|
3331
3463
|
}
|
|
3332
3464
|
}
|
|
3333
|
-
})
|
|
3334
|
-
|
|
3465
|
+
}, (err) => {
|
|
3466
|
+
errorRequest(item, err);
|
|
3467
|
+
}, 4);
|
|
3335
3468
|
}
|
|
3336
3469
|
}
|
|
3337
3470
|
else if (!this.canRead(uri)) {
|
|
@@ -3348,12 +3481,187 @@ class FileManager extends core_1.Host {
|
|
|
3348
3481
|
});
|
|
3349
3482
|
}
|
|
3350
3483
|
else {
|
|
3351
|
-
fileReceived(item, localUri);
|
|
3484
|
+
fileReceived(item, localUri, null);
|
|
3352
3485
|
}
|
|
3353
3486
|
}
|
|
3354
3487
|
}
|
|
3355
3488
|
this.cleared = true;
|
|
3356
3489
|
}
|
|
3490
|
+
updateProgress(name, ...args) {
|
|
3491
|
+
if (name === 'request' && STDOUT_CURSOR) {
|
|
3492
|
+
const processTask = this[kScheduler];
|
|
3493
|
+
const status = processTask.status;
|
|
3494
|
+
const id = args[0];
|
|
3495
|
+
const found = status.find(item => item.id === id);
|
|
3496
|
+
if (!found) {
|
|
3497
|
+
return;
|
|
3498
|
+
}
|
|
3499
|
+
const received = args[1];
|
|
3500
|
+
const total = args[2];
|
|
3501
|
+
if (received === 0 && total === 0) {
|
|
3502
|
+
if (Array.isArray(args[3])) {
|
|
3503
|
+
found.dataTime = args[3];
|
|
3504
|
+
}
|
|
3505
|
+
return;
|
|
3506
|
+
}
|
|
3507
|
+
found.receivedBytes = received;
|
|
3508
|
+
found.totalBytes = total;
|
|
3509
|
+
const currentTime = Date.now();
|
|
3510
|
+
if (received === total || received === 0) {
|
|
3511
|
+
found.endTime = currentTime;
|
|
3512
|
+
found.previousBytes = 0;
|
|
3513
|
+
}
|
|
3514
|
+
else if (processTask.updated + 100 > currentTime) {
|
|
3515
|
+
return;
|
|
3516
|
+
}
|
|
3517
|
+
const length = status.length;
|
|
3518
|
+
const logCurrent = (0, types_1.getLogCurrent)();
|
|
3519
|
+
const redraw = processTask.logCurrent !== logCurrent;
|
|
3520
|
+
const formatSegment = (endTime, value) => endTime ? value : chalk.bgGray.black(value);
|
|
3521
|
+
if (redraw || length > processTask.length) {
|
|
3522
|
+
this.pauseLog();
|
|
3523
|
+
processTask.logCurrent = logCurrent;
|
|
3524
|
+
PROCESS_STDOUT.write('\n'.repeat(redraw ? length : length - processTask.length));
|
|
3525
|
+
processTask.length = length;
|
|
3526
|
+
}
|
|
3527
|
+
PROCESS_STDOUT.moveCursor(0, -length);
|
|
3528
|
+
processTask.updated = currentTime;
|
|
3529
|
+
const titleWidth = processTask.titleWidth;
|
|
3530
|
+
let finished = true;
|
|
3531
|
+
for (let i = 0; i < length; ++i) {
|
|
3532
|
+
const item = status[i];
|
|
3533
|
+
const { receivedBytes, endTime } = item;
|
|
3534
|
+
if (!endTime) {
|
|
3535
|
+
finished = false;
|
|
3536
|
+
}
|
|
3537
|
+
if (item.previousBytes === receivedBytes && !redraw) {
|
|
3538
|
+
PROCESS_STDOUT.moveCursor(0, 1);
|
|
3539
|
+
continue;
|
|
3540
|
+
}
|
|
3541
|
+
if (item.output) {
|
|
3542
|
+
PROCESS_STDOUT.write(item.output);
|
|
3543
|
+
continue;
|
|
3544
|
+
}
|
|
3545
|
+
let percent = 0, bars = 0;
|
|
3546
|
+
item.previousBytes = receivedBytes;
|
|
3547
|
+
if (endTime) {
|
|
3548
|
+
if (receivedBytes === item.totalBytes) {
|
|
3549
|
+
bars = -1;
|
|
3550
|
+
}
|
|
3551
|
+
}
|
|
3552
|
+
else if (receivedBytes > item.totalBytes) {
|
|
3553
|
+
item.totalBytes = receivedBytes;
|
|
3554
|
+
percent = 99;
|
|
3555
|
+
bars = 24;
|
|
3556
|
+
}
|
|
3557
|
+
else {
|
|
3558
|
+
percent = (receivedBytes / item.totalBytes) * 100;
|
|
3559
|
+
bars = Math.trunc(percent / 4);
|
|
3560
|
+
}
|
|
3561
|
+
const diffTime = currentTime - item.startTime;
|
|
3562
|
+
const time = formatSegment(endTime, formatMinutes(diffTime).padStart(titleWidth));
|
|
3563
|
+
const size = formatSegment(endTime, (0, types_1.formatSize)(receivedBytes, { unit: receivedBytes < 1048576 ? 'KB' : receivedBytes < 1073741824 ? 'MB' : 'GB', unitSeparator: ' ' }).padStart(9));
|
|
3564
|
+
const speed = formatSegment(endTime, (0, util_1.getTransferRate)(receivedBytes, item.dataTime ? (0, types_1.convertTime)(process.hrtime(item.dataTime), false) * 1000 : diffTime, ' ').padStart(10));
|
|
3565
|
+
const output = time + ` ${chalk.blackBright(':')} ${endTime && bars !== -1 ? chalk.bgBlack.gray(item.url) : item.url}| ${size} | ${speed} |${bars === -1 ? chalk.bgBlack.green.bold('>'.repeat(25)) : (bars > 0 ? chalk.yellow('>'.repeat(bars)) : '') + (25 - bars > 0 ? chalk.gray((endTime ? '#' : '-').repeat(25 - bars)) : '')}| ${endTime ? bars === -1 ? chalk.bold('100%') : chalk.red('ERR!') : Math.trunc(percent).toString().padStart(3) + '%'}\n`;
|
|
3566
|
+
PROCESS_STDOUT.write(output);
|
|
3567
|
+
if (endTime) {
|
|
3568
|
+
item.output = output;
|
|
3569
|
+
}
|
|
3570
|
+
}
|
|
3571
|
+
if (finished) {
|
|
3572
|
+
this.resumeLog();
|
|
3573
|
+
processTask.reset();
|
|
3574
|
+
}
|
|
3575
|
+
}
|
|
3576
|
+
}
|
|
3577
|
+
scheduleTask(url, data, thenCallback, catchCallback, priority = 0) {
|
|
3578
|
+
if (typeof thenCallback === 'number') {
|
|
3579
|
+
priority = thenCallback;
|
|
3580
|
+
thenCallback = undefined;
|
|
3581
|
+
}
|
|
3582
|
+
const scheduler = this[kScheduler];
|
|
3583
|
+
if (scheduler.count === scheduler.limit || this.finalizeState !== 0) {
|
|
3584
|
+
scheduler.queue.push([url, data, thenCallback, catchCallback, typeof priority === 'number' ? priority : 0]);
|
|
3585
|
+
scheduler.queue.sort((a, b) => b[4] - a[4]);
|
|
3586
|
+
return Promise.resolve(0);
|
|
3587
|
+
}
|
|
3588
|
+
const nextTask = (pid) => {
|
|
3589
|
+
if (pid > 0) {
|
|
3590
|
+
const found = scheduler.status.find(item => item.id === pid);
|
|
3591
|
+
if (found) {
|
|
3592
|
+
found.endTime || (found.endTime = Date.now());
|
|
3593
|
+
}
|
|
3594
|
+
}
|
|
3595
|
+
--scheduler.count;
|
|
3596
|
+
if (scheduler.queue.length) {
|
|
3597
|
+
const params = scheduler.queue.pop();
|
|
3598
|
+
this.scheduleTask(...params);
|
|
3599
|
+
}
|
|
3600
|
+
else if (scheduler.transform.length) {
|
|
3601
|
+
do {
|
|
3602
|
+
const args = scheduler.transform.shift();
|
|
3603
|
+
this.transformAsset(args[0], args[1], true);
|
|
3604
|
+
} while (scheduler.transform.length && scheduler.queue.length === 0);
|
|
3605
|
+
}
|
|
3606
|
+
};
|
|
3607
|
+
let id = 0, target;
|
|
3608
|
+
if ((0, types_1.isPlainObject)(data)) {
|
|
3609
|
+
++scheduler.count;
|
|
3610
|
+
if ('pathname' in data || 'binOpts' in data) {
|
|
3611
|
+
target = this.fetchFiles(url, data);
|
|
3612
|
+
}
|
|
3613
|
+
else {
|
|
3614
|
+
if (this.hasLog('progress') && STDOUT_CURSOR) {
|
|
3615
|
+
id = ++scheduler.id;
|
|
3616
|
+
data.progressId = id;
|
|
3617
|
+
}
|
|
3618
|
+
target = this.fetchBuffer(url, data);
|
|
3619
|
+
}
|
|
3620
|
+
}
|
|
3621
|
+
else {
|
|
3622
|
+
if (Buffer.isBuffer(data)) {
|
|
3623
|
+
data = stream.Readable.from(data);
|
|
3624
|
+
}
|
|
3625
|
+
else {
|
|
3626
|
+
if (data instanceof URL && data.protocol === 'file:') {
|
|
3627
|
+
data = require('url').fileURLToPath(data);
|
|
3628
|
+
}
|
|
3629
|
+
if ((0, types_1.isString)(data) && core_1.Host.isPath(data, true) && this.canRead(data)) {
|
|
3630
|
+
data = fs.createReadStream(data);
|
|
3631
|
+
}
|
|
3632
|
+
}
|
|
3633
|
+
if (data instanceof stream.Readable) {
|
|
3634
|
+
++scheduler.count;
|
|
3635
|
+
target = new Promise((resolve, reject) => {
|
|
3636
|
+
const writeable = fs.createWriteStream(url);
|
|
3637
|
+
writeable.on('finish', () => {
|
|
3638
|
+
resolve(1);
|
|
3639
|
+
});
|
|
3640
|
+
writeable.on('error', reject);
|
|
3641
|
+
data.on('error', reject);
|
|
3642
|
+
data.pipe(writeable);
|
|
3643
|
+
});
|
|
3644
|
+
}
|
|
3645
|
+
}
|
|
3646
|
+
if (target) {
|
|
3647
|
+
if (thenCallback) {
|
|
3648
|
+
target.then(thenCallback);
|
|
3649
|
+
}
|
|
3650
|
+
if (catchCallback) {
|
|
3651
|
+
target.catch(catchCallback);
|
|
3652
|
+
}
|
|
3653
|
+
if (id > 0) {
|
|
3654
|
+
scheduler.add(id, url.toString());
|
|
3655
|
+
}
|
|
3656
|
+
return target.finally(nextTask.bind(this, id));
|
|
3657
|
+
}
|
|
3658
|
+
return Promise.resolve(-1);
|
|
3659
|
+
}
|
|
3660
|
+
setTaskLimit(value) {
|
|
3661
|
+
if ((value = Math.trunc(value)) > 0) {
|
|
3662
|
+
this[kScheduler].limit = Math.min(value, PROCESS_SUB_LIMIT);
|
|
3663
|
+
}
|
|
3664
|
+
}
|
|
3357
3665
|
async finalizeCompress(assets) {
|
|
3358
3666
|
const compress = this.Compress;
|
|
3359
3667
|
if (!(assets.length && compress)) {
|
|
@@ -3362,7 +3670,7 @@ class FileManager extends core_1.Host {
|
|
|
3362
3670
|
const tasks = [];
|
|
3363
3671
|
for (const item of assets) {
|
|
3364
3672
|
const localUri = item.localUri;
|
|
3365
|
-
if (!(localUri && image_1.
|
|
3673
|
+
if (!(localUri && image_1.isBinary(item.mimeType))) {
|
|
3366
3674
|
continue;
|
|
3367
3675
|
}
|
|
3368
3676
|
const files = [localUri];
|
|
@@ -3391,7 +3699,7 @@ class FileManager extends core_1.Host {
|
|
|
3391
3699
|
item.buffer = result;
|
|
3392
3700
|
}
|
|
3393
3701
|
})
|
|
3394
|
-
.catch(err => this.writeFail(["Unable to compress file", path.basename(file)], err, { type: 8, startTime: options.startTime })));
|
|
3702
|
+
.catch((err) => this.writeFail(["Unable to compress file", path.basename(file)], err, { type: 8, startTime: options.startTime })));
|
|
3395
3703
|
}
|
|
3396
3704
|
}
|
|
3397
3705
|
}
|
|
@@ -3402,7 +3710,7 @@ class FileManager extends core_1.Host {
|
|
|
3402
3710
|
async finalizeDocument() {
|
|
3403
3711
|
for (const { instance, constructor } of this.Document) {
|
|
3404
3712
|
if (constructor.finalize && instance.assets.length) {
|
|
3405
|
-
await constructor.finalize.call(this, instance).catch(err => rejectModule.call(instance, err, 4));
|
|
3713
|
+
await constructor.finalize.call(this, instance).catch((err) => rejectModule.call(instance, err, 4));
|
|
3406
3714
|
if (this.aborted) {
|
|
3407
3715
|
return;
|
|
3408
3716
|
}
|
|
@@ -3414,7 +3722,7 @@ class FileManager extends core_1.Host {
|
|
|
3414
3722
|
for (const { instance, constructor } of this.Task) {
|
|
3415
3723
|
const items = assets.filter(item => item.tasks.find(data => data.handler === instance.moduleName));
|
|
3416
3724
|
if (items.length) {
|
|
3417
|
-
await
|
|
3725
|
+
await constructor.finalize.call(this, instance, items).catch((err) => rejectModule.call(instance, err, 4));
|
|
3418
3726
|
if (this.aborted) {
|
|
3419
3727
|
return;
|
|
3420
3728
|
}
|
|
@@ -3425,7 +3733,7 @@ class FileManager extends core_1.Host {
|
|
|
3425
3733
|
async finalizeCloud() {
|
|
3426
3734
|
const cloud = this.Cloud;
|
|
3427
3735
|
if (cloud) {
|
|
3428
|
-
return cloud_1.
|
|
3736
|
+
return cloud_1.finalize.call(this, cloud).catch((err) => rejectModule.call(cloud, err, 64));
|
|
3429
3737
|
}
|
|
3430
3738
|
}
|
|
3431
3739
|
async finalizeCleanup() {
|
|
@@ -3440,13 +3748,13 @@ class FileManager extends core_1.Host {
|
|
|
3440
3748
|
}
|
|
3441
3749
|
for (const { instance, constructor } of this.Document) {
|
|
3442
3750
|
if (constructor.cleanup && instance.assets.length) {
|
|
3443
|
-
await constructor.cleanup.call(this, instance).catch(err => rejectModule.call(instance, err, 4));
|
|
3751
|
+
await constructor.cleanup.call(this, instance).catch((err) => rejectModule.call(instance, err, 4));
|
|
3444
3752
|
}
|
|
3445
3753
|
}
|
|
3446
3754
|
}
|
|
3447
3755
|
async finalize() {
|
|
3448
3756
|
if (this.aborted) {
|
|
3449
|
-
return
|
|
3757
|
+
return (0, types_1.createAbortError)(true);
|
|
3450
3758
|
}
|
|
3451
3759
|
if (LOG_TIMEELAPSED) {
|
|
3452
3760
|
this.writeTimeElapsed('READY', 'Finalizing assets...', this.startTime, { ...core_1.Host.LOG_STYLE_WARN, showCpu: true });
|
|
@@ -3461,9 +3769,9 @@ class FileManager extends core_1.Host {
|
|
|
3461
3769
|
};
|
|
3462
3770
|
for (const [file, output] of this.filesToCompare) {
|
|
3463
3771
|
const localUri = file.localUri;
|
|
3464
|
-
let minFile = localUri, minSize = (0,
|
|
3772
|
+
let minFile = localUri, minSize = (0, util_1.getSize)(minFile);
|
|
3465
3773
|
for (const other of output) {
|
|
3466
|
-
const size = (0,
|
|
3774
|
+
const size = (0, util_1.getSize)(other);
|
|
3467
3775
|
if (minSize === 0 || size > 0 && size < minSize) {
|
|
3468
3776
|
filesToRemove.add(minFile);
|
|
3469
3777
|
minFile = other;
|
|
@@ -3478,13 +3786,13 @@ class FileManager extends core_1.Host {
|
|
|
3478
3786
|
}
|
|
3479
3787
|
}
|
|
3480
3788
|
removeFiles();
|
|
3481
|
-
await this.finalizeCompress(this.assets.filter(item => item.compress && !ignoreAsset(item))).catch(err => rejectModule.call(this, err, 8));
|
|
3789
|
+
await this.finalizeCompress(this.assets.filter(item => item.compress && !ignoreAsset(item))).catch((err) => rejectModule.call(this, err, 8));
|
|
3482
3790
|
if (this.aborted) {
|
|
3483
|
-
return
|
|
3791
|
+
return (0, types_1.createAbortError)(true);
|
|
3484
3792
|
}
|
|
3485
|
-
await this.finalizeDocument().catch(err => rejectModule.call(this, err, 4));
|
|
3793
|
+
await this.finalizeDocument().catch((err) => rejectModule.call(this, err, 4));
|
|
3486
3794
|
if (this.aborted) {
|
|
3487
|
-
return
|
|
3795
|
+
return (0, types_1.createAbortError)(true);
|
|
3488
3796
|
}
|
|
3489
3797
|
for (const item of this.assets) {
|
|
3490
3798
|
if (item.sourceUTF8 && !ignoreAsset(item)) {
|
|
@@ -3497,9 +3805,9 @@ class FileManager extends core_1.Host {
|
|
|
3497
3805
|
}
|
|
3498
3806
|
}
|
|
3499
3807
|
removeFiles();
|
|
3500
|
-
await this.finalizeTask(this.taskAssets.filter(item => item.tasks?.find(data => !data.preceding) && item.localUri && this.has(item.localUri) && !ignoreAsset(item))).catch(err => rejectModule.call(this, err, 4));
|
|
3808
|
+
await this.finalizeTask(this.taskAssets.filter(item => item.tasks?.find(data => !data.preceding) && item.localUri && this.has(item.localUri) && !ignoreAsset(item))).catch((err) => rejectModule.call(this, err, 4));
|
|
3501
3809
|
if (this.aborted) {
|
|
3502
|
-
return
|
|
3810
|
+
return (0, types_1.createAbortError)(true);
|
|
3503
3811
|
}
|
|
3504
3812
|
removeFiles();
|
|
3505
3813
|
for (const item of this.assets) {
|
|
@@ -3513,9 +3821,9 @@ class FileManager extends core_1.Host {
|
|
|
3513
3821
|
}
|
|
3514
3822
|
}
|
|
3515
3823
|
removeFiles();
|
|
3516
|
-
await this.finalizeCloud().catch(err => rejectModule.call(this, err, 64));
|
|
3824
|
+
await this.finalizeCloud().catch((err) => rejectModule.call(this, err, 64));
|
|
3517
3825
|
if (this.aborted) {
|
|
3518
|
-
return
|
|
3826
|
+
return (0, types_1.createAbortError)(true);
|
|
3519
3827
|
}
|
|
3520
3828
|
removeFiles();
|
|
3521
3829
|
if (this.Compress) {
|
|
@@ -3550,32 +3858,32 @@ class FileManager extends core_1.Host {
|
|
|
3550
3858
|
}
|
|
3551
3859
|
}
|
|
3552
3860
|
}
|
|
3553
|
-
await this.finalizeCleanup().catch(err => rejectModule.call(this, err, 1));
|
|
3861
|
+
await this.finalizeCleanup().catch((err) => rejectModule.call(this, err, 1));
|
|
3554
3862
|
removeFiles();
|
|
3555
3863
|
if (LOG_TIMEELAPSED) {
|
|
3556
|
-
const [
|
|
3864
|
+
const [http, disk, cache] = this[kDownloadStats];
|
|
3557
3865
|
const errorCount = this.errorCount;
|
|
3558
3866
|
const message = [];
|
|
3559
|
-
if (errorCount) {
|
|
3867
|
+
if (errorCount > 0) {
|
|
3560
3868
|
message.push('ERROR ' + errorCount);
|
|
3561
3869
|
}
|
|
3562
|
-
let [size, count] =
|
|
3563
|
-
if (count) {
|
|
3564
|
-
message.push('HTTP ' + (count > 1 ? count + '/' : '') + (0, types_1.formatSize)(size));
|
|
3870
|
+
let [size, count] = http;
|
|
3871
|
+
if (count > 0) {
|
|
3872
|
+
message.push('HTTP ' + (count > 1 ? count + ' / ' : '') + (0, types_1.formatSize)(size));
|
|
3565
3873
|
}
|
|
3566
|
-
[size, count] =
|
|
3567
|
-
if (count) {
|
|
3568
|
-
message.push('DISK ' + (count > 1 ? count + '/' : '') + (0, types_1.formatSize)(size));
|
|
3874
|
+
[size, count] = disk;
|
|
3875
|
+
if (count > 0) {
|
|
3876
|
+
message.push('DISK ' + (count > 1 ? count + ' / ' : '') + (0, types_1.formatSize)(size));
|
|
3569
3877
|
}
|
|
3570
|
-
[size, count] =
|
|
3571
|
-
if (count) {
|
|
3572
|
-
message.push('CACHE ' + (count > 1 ? count + '/' : '') + (0, types_1.formatSize)(size));
|
|
3878
|
+
[size, count] = cache;
|
|
3879
|
+
if (count > 0) {
|
|
3880
|
+
message.push('CACHE ' + (count > 1 ? count + ' / ' : '') + (0, types_1.formatSize)(size));
|
|
3573
3881
|
}
|
|
3574
3882
|
if (MEMORY.SIZE > 0) {
|
|
3575
3883
|
const bufferCount = Object.keys(MEMORY.CACHE).length;
|
|
3576
|
-
message.push('BUFFER ' + (bufferCount > 1 ? bufferCount + '/' : '') + (0, types_1.formatSize)(MEMORY.SIZE));
|
|
3884
|
+
message.push('BUFFER ' + (bufferCount > 1 ? bufferCount + ' / ' : '') + (0, types_1.formatSize)(MEMORY.SIZE));
|
|
3577
3885
|
}
|
|
3578
|
-
this.writeTimeElapsed('CLOSE', ['No further modifications', message.join('
|
|
3886
|
+
this.writeTimeElapsed('CLOSE', ['No further modifications', message.length ? ` ${message.join(chalk.blackBright(' )( '))} ` : ''], startTime, { ...core_1.Host.LOG_STYLE_WARN });
|
|
3579
3887
|
}
|
|
3580
3888
|
this.Watch?.start(this.assets);
|
|
3581
3889
|
}
|
|
@@ -3609,13 +3917,29 @@ class FileManager extends core_1.Host {
|
|
|
3609
3917
|
get cleared() {
|
|
3610
3918
|
return this[kCleared];
|
|
3611
3919
|
}
|
|
3920
|
+
set finalizeState(value) {
|
|
3921
|
+
if (value >= 0 && value <= 6) {
|
|
3922
|
+
this[kFinalizedState] = value;
|
|
3923
|
+
if (value === 0) {
|
|
3924
|
+
const { count, limit, queue } = this[kScheduler];
|
|
3925
|
+
const available = limit - count;
|
|
3926
|
+
if (queue.length > 0 && available > 0) {
|
|
3927
|
+
const start = Math.max(queue.length - available, 0);
|
|
3928
|
+
const items = queue.slice(start);
|
|
3929
|
+
queue.splice(queue.length - items.length);
|
|
3930
|
+
items.forEach(params => {
|
|
3931
|
+
this.scheduleTask(...params);
|
|
3932
|
+
});
|
|
3933
|
+
}
|
|
3934
|
+
}
|
|
3935
|
+
}
|
|
3936
|
+
}
|
|
3937
|
+
get finalizeState() {
|
|
3938
|
+
return this[kFinalizedState];
|
|
3939
|
+
}
|
|
3612
3940
|
get errorCount() {
|
|
3613
3941
|
return Array.from(this.modules).reduce((a, b) => a + b.errors.length, this.errors.length);
|
|
3614
3942
|
}
|
|
3615
3943
|
}
|
|
3616
|
-
exports.default = FileManager;
|
|
3617
3944
|
|
|
3618
|
-
|
|
3619
|
-
module.exports = exports.default;
|
|
3620
|
-
module.exports.default = exports.default;
|
|
3621
|
-
}
|
|
3945
|
+
module.exports = FileManager;
|