@e-mc/file-manager 0.13.10 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -11
- package/index.d.ts +2 -2
- package/index.js +404 -348
- package/package.json +11 -14
package/index.js
CHANGED
|
@@ -1,22 +1,23 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
|
|
3
|
-
const
|
|
4
|
-
const
|
|
5
|
-
const
|
|
6
|
-
const
|
|
7
|
-
const chalk = require(
|
|
8
|
-
const
|
|
9
|
-
const
|
|
10
|
-
const
|
|
11
|
-
const
|
|
12
|
-
const
|
|
13
|
-
const
|
|
14
|
-
const
|
|
15
|
-
const
|
|
16
|
-
const
|
|
17
|
-
const
|
|
18
|
-
const
|
|
19
|
-
const
|
|
2
|
+
|
|
3
|
+
const path = require('node:path');
|
|
4
|
+
const fs = require('node:fs');
|
|
5
|
+
const stream = require('node:stream');
|
|
6
|
+
const pm = require('picomatch');
|
|
7
|
+
const chalk = require('chalk');
|
|
8
|
+
const { fileURLToPath } = require('node:url');
|
|
9
|
+
const { stripVTControlCharacters } = require('node:util');
|
|
10
|
+
const { structuredPatch } = require('diff');
|
|
11
|
+
const { DOWNLOAD_TYPE, FILE_TYPE, THRESHOLD, asExt, cloneFlag, convertTime, createAbortError, errorMessage, errorValue, existsFlag, formatSize, formatTime, getAlgorithm, getEncoding, getLogCurrent, getTempDir, hasBit, hasGlob, ignoreFlag, incrementUUID, isArray, isEmpty, isError, isErrorCode, isFunction, isObject, isPlainObject, isString, mainFlag, parseExpires, parseTime, renameExt, requireESM, setLogCurrent, usingFlag, watchFlag } = require('@e-mc/types');
|
|
12
|
+
const { AbortComponent, Host, Permission } = require('@e-mc/core');
|
|
13
|
+
const { isEqual, setInitialValue } = require('@e-mc/document/asset');
|
|
14
|
+
const { concatString, getNewline } = require('@e-mc/document/util');
|
|
15
|
+
const { asFloat, asInt, byteLength, checkRetryable, fromSeconds, fromURL, getSize, getTransferRate, hasSameStat, hasSize, parseHeader } = require('@e-mc/request/util');
|
|
16
|
+
const Request = require('@e-mc/request');
|
|
17
|
+
const Image = require('@e-mc/image');
|
|
18
|
+
const Watch = require('@e-mc/watch');
|
|
19
|
+
const Compress = require('@e-mc/compress');
|
|
20
|
+
const Cloud = require('@e-mc/cloud');
|
|
20
21
|
const kFileManager = Symbol.for('filemanager:constructor');
|
|
21
22
|
const PROCESS_STDOUT = process.stdout;
|
|
22
23
|
const PROCESS_STDIN = process.stdin;
|
|
@@ -27,7 +28,7 @@ const CACHE_ETAG = new Map();
|
|
|
27
28
|
const DISK = {
|
|
28
29
|
ENABLED: false,
|
|
29
30
|
EXPIRES: 0,
|
|
30
|
-
LIMIT:
|
|
31
|
+
LIMIT: formatSize("1gb"),
|
|
31
32
|
INCLUDE: [],
|
|
32
33
|
EXCLUDE: []
|
|
33
34
|
};
|
|
@@ -36,8 +37,8 @@ const MEMORY = {
|
|
|
36
37
|
CACHE: new Map(),
|
|
37
38
|
SIZE: 0,
|
|
38
39
|
EXPIRES: 0,
|
|
39
|
-
LIMIT:
|
|
40
|
-
LIMIT_ALL:
|
|
40
|
+
LIMIT: formatSize("100mb"),
|
|
41
|
+
LIMIT_ALL: formatSize("512mb"),
|
|
41
42
|
INCLUDE: [],
|
|
42
43
|
EXCLUDE: [],
|
|
43
44
|
PURGE: 0.25,
|
|
@@ -70,7 +71,7 @@ let SESSION_ID = 0;
|
|
|
70
71
|
let SESSION_LIMIT = 1000;
|
|
71
72
|
let ASSET_ID = 0;
|
|
72
73
|
let RETRY_LIMIT = 10;
|
|
73
|
-
let PROCESS_SUB_LIMIT =
|
|
74
|
+
let PROCESS_SUB_LIMIT = Host.availableParallelism(2);
|
|
74
75
|
let LOG_TIMEELAPSED = true;
|
|
75
76
|
let LOG_TIMEPROCESS = true;
|
|
76
77
|
const HTTP_CLIENT = {
|
|
@@ -93,7 +94,7 @@ function parseTimeout(value) {
|
|
|
93
94
|
if (+value === 0) {
|
|
94
95
|
return 0;
|
|
95
96
|
}
|
|
96
|
-
if ((value =
|
|
97
|
+
if ((value = parseTime(value)) > 0) {
|
|
97
98
|
return value;
|
|
98
99
|
}
|
|
99
100
|
return -1;
|
|
@@ -102,7 +103,7 @@ function withinSizeRange(uri, value, pattern) {
|
|
|
102
103
|
const match = (pattern || /\(\s*(\d+)\s*(?:,\s*(\d+|\*)\s*)?\)/).exec(value);
|
|
103
104
|
const [minSize, maxSize] = match && !match[1] ? [+match[2], !match[3] || match[3] === '*' ? Infinity : +match[3]] : [0, Infinity];
|
|
104
105
|
if (minSize > 0 || maxSize < Infinity) {
|
|
105
|
-
const fileSize =
|
|
106
|
+
const fileSize = getSize(uri);
|
|
106
107
|
if (fileSize === 0 || fileSize < minSize || fileSize > maxSize) {
|
|
107
108
|
return false;
|
|
108
109
|
}
|
|
@@ -112,7 +113,7 @@ function withinSizeRange(uri, value, pattern) {
|
|
|
112
113
|
function bundleTorrent(host, files, mimeType, encoding) {
|
|
113
114
|
let output = '';
|
|
114
115
|
for (const file of files) {
|
|
115
|
-
if (
|
|
116
|
+
if (Host.lookupMime(file) === mimeType) {
|
|
116
117
|
try {
|
|
117
118
|
output += fs.readFileSync(file, encoding);
|
|
118
119
|
}
|
|
@@ -126,14 +127,14 @@ function bundleTorrent(host, files, mimeType, encoding) {
|
|
|
126
127
|
function recurseDir(output, subDirs, options) {
|
|
127
128
|
const { ignore, ignoreRoot, sortBy, recursive } = options;
|
|
128
129
|
const fromDir = subDirs.length > 1 ? path.posix.join(...subDirs.slice(1)) : null;
|
|
129
|
-
if (ignoreRoot.length > 0 && fromDir && pm.isMatch(fromDir + '/', ignoreRoot,
|
|
130
|
+
if (ignoreRoot.length > 0 && fromDir && pm.isMatch(fromDir + '/', ignoreRoot, Host.PLATFORM_WIN32 ? { nocase: true } : undefined)) {
|
|
130
131
|
return output;
|
|
131
132
|
}
|
|
132
|
-
const ignoreGlob = ignore.length > 0 ? pm(ignore,
|
|
133
|
+
const ignoreGlob = ignore.length > 0 ? pm(ignore, Host.PLATFORM_WIN32 ? { nocase: true, matchBase: true } : { matchBase: true }) : null;
|
|
133
134
|
const baseDir = path.join(...subDirs);
|
|
134
135
|
const items = fs.readdirSync(baseDir, { withFileTypes: true })
|
|
135
136
|
.filter(item => {
|
|
136
|
-
if (ignoreGlob
|
|
137
|
+
if (ignoreGlob?.(fromDir ? path.posix.join(fromDir, item.name) : item.name)) {
|
|
137
138
|
return false;
|
|
138
139
|
}
|
|
139
140
|
return item.name;
|
|
@@ -152,12 +153,12 @@ function recurseDir(output, subDirs, options) {
|
|
|
152
153
|
for (const item of items) {
|
|
153
154
|
if (!item.isDirectory()) {
|
|
154
155
|
const pathname = path.join(baseDir, item.name);
|
|
155
|
-
if (item.isSymbolicLink() &&
|
|
156
|
+
if (item.isSymbolicLink() && Host.isDir(pathname) || ignore.length > 0 && !hasPath(ignore, pathname)) {
|
|
156
157
|
continue;
|
|
157
158
|
}
|
|
158
159
|
output.push(pathname);
|
|
159
160
|
}
|
|
160
|
-
else if (recursive && (subDirs.length > 1 || (
|
|
161
|
+
else if (recursive && (subDirs.length > 1 || !hasPath(ignoreRoot, item.name))) {
|
|
161
162
|
recurseDir(output, subDirs.concat(item.name), options);
|
|
162
163
|
}
|
|
163
164
|
}
|
|
@@ -171,10 +172,10 @@ function createMatchNegate(patterns, options) {
|
|
|
171
172
|
for (let value of patterns) {
|
|
172
173
|
let invert = false;
|
|
173
174
|
if (value.startsWith('!')) {
|
|
174
|
-
value = value.
|
|
175
|
+
value = value.slice(1);
|
|
175
176
|
invert = true;
|
|
176
177
|
}
|
|
177
|
-
value =
|
|
178
|
+
value = Permission.toPosix(value);
|
|
178
179
|
const match = pm(value, getPmOptions(value, options));
|
|
179
180
|
if (path.isAbsolute(value)) {
|
|
180
181
|
(invert ? negateAbs : baseAbs).push(match);
|
|
@@ -190,30 +191,30 @@ function isMatch(value, globs) {
|
|
|
190
191
|
}
|
|
191
192
|
function filterPaths(rootDir, values, options = {}) {
|
|
192
193
|
let { include, exclude } = options;
|
|
193
|
-
rootDir =
|
|
194
|
+
rootDir = Host.normalizePath(rootDir, 2);
|
|
194
195
|
if (include) {
|
|
195
|
-
if (
|
|
196
|
+
if (isString(include)) {
|
|
196
197
|
include = [include];
|
|
197
198
|
}
|
|
198
|
-
if (
|
|
199
|
+
if (isArray(include)) {
|
|
199
200
|
const [glob, globAbs] = createMatchNegate(include, options);
|
|
200
201
|
for (let i = 0; i < values.length; ++i) {
|
|
201
202
|
const value = values[i];
|
|
202
|
-
if (!isMatch(value, globAbs) && !isMatch(value.
|
|
203
|
+
if (!isMatch(value, globAbs) && !isMatch(value.slice(rootDir.length), glob)) {
|
|
203
204
|
values.splice(i--, 1);
|
|
204
205
|
}
|
|
205
206
|
}
|
|
206
207
|
}
|
|
207
208
|
}
|
|
208
209
|
if (exclude) {
|
|
209
|
-
if (
|
|
210
|
+
if (isString(exclude)) {
|
|
210
211
|
exclude = [exclude];
|
|
211
212
|
}
|
|
212
|
-
if (
|
|
213
|
+
if (isArray(exclude)) {
|
|
213
214
|
const [glob, globAbs] = createMatchNegate(exclude, options);
|
|
214
215
|
for (let i = 0; i < values.length; ++i) {
|
|
215
216
|
const value = values[i];
|
|
216
|
-
if (isMatch(value, globAbs) || isMatch(value.
|
|
217
|
+
if (isMatch(value, globAbs) || isMatch(value.slice(rootDir.length), glob)) {
|
|
217
218
|
values.splice(i--, 1);
|
|
218
219
|
}
|
|
219
220
|
}
|
|
@@ -228,7 +229,7 @@ function formatMinutes(elapsed) {
|
|
|
228
229
|
if (elapsed < 60000) {
|
|
229
230
|
return (elapsed / 1000).toPrecision(3) + 's';
|
|
230
231
|
}
|
|
231
|
-
return
|
|
232
|
+
return formatTime(elapsed, ':');
|
|
232
233
|
}
|
|
233
234
|
function clearQueue(data, attr) {
|
|
234
235
|
if (attr && data[attr]) {
|
|
@@ -239,7 +240,7 @@ function clearQueue(data, attr) {
|
|
|
239
240
|
}
|
|
240
241
|
}
|
|
241
242
|
function hasEtag(item, cacheable, fallback) {
|
|
242
|
-
if (!cacheable || !
|
|
243
|
+
if (!cacheable || !existsFlag(item)) {
|
|
243
244
|
return false;
|
|
244
245
|
}
|
|
245
246
|
switch (item.incremental) {
|
|
@@ -262,7 +263,7 @@ function equalAddress(value, other) {
|
|
|
262
263
|
const b = decodeURIComponent(other);
|
|
263
264
|
return a === b || a === other || value === b;
|
|
264
265
|
}
|
|
265
|
-
function processHeaders(
|
|
266
|
+
function processHeaders(headers, item, lastModified, mainEtag) {
|
|
266
267
|
const contentLength = parseInt(headers['content-length']);
|
|
267
268
|
if (contentLength > 0) {
|
|
268
269
|
item.contentLength = contentLength;
|
|
@@ -281,6 +282,36 @@ function processHeaders(item, headers, lastModified, mainEtag) {
|
|
|
281
282
|
return item.lastModified = headers['last-modified'];
|
|
282
283
|
}
|
|
283
284
|
}
|
|
285
|
+
function processCacheHeaders(headers, noCache, noStore) {
|
|
286
|
+
if (noStore) {
|
|
287
|
+
return NaN;
|
|
288
|
+
}
|
|
289
|
+
const cacheControl = parseHeader(headers, 'cache-control');
|
|
290
|
+
if (cacheControl) {
|
|
291
|
+
if (cacheControl.includes('no-store')) {
|
|
292
|
+
return -1;
|
|
293
|
+
}
|
|
294
|
+
const maxAge = cacheControl.find(value => value.startsWith('max-age='));
|
|
295
|
+
if (maxAge && (!noCache || cacheControl.includes('public') || cacheControl.includes('must-revalidate'))) {
|
|
296
|
+
const age = parseHeader(headers, 'age');
|
|
297
|
+
let result = parseInt(maxAge.split('=').pop()) * 1000;
|
|
298
|
+
if (age) {
|
|
299
|
+
result -= age;
|
|
300
|
+
if (result < 0 && noCache) {
|
|
301
|
+
return -1;
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
return result > 0 ? result : 0;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
if (!noCache) {
|
|
308
|
+
const expires = parseHeader(headers, 'expires');
|
|
309
|
+
if (expires && !isNaN(expires)) {
|
|
310
|
+
const date = parseHeader(headers, 'date') || Date.now();
|
|
311
|
+
return expires - date;
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
}
|
|
284
315
|
function setContent(item, buffer) {
|
|
285
316
|
if (typeof buffer === 'string') {
|
|
286
317
|
item.sourceUTF8 = buffer;
|
|
@@ -298,38 +329,23 @@ function unsetContent(item) {
|
|
|
298
329
|
}
|
|
299
330
|
}
|
|
300
331
|
async function doVerifyChecksum(root, from, options, nested) {
|
|
301
|
-
const { digestEncoding
|
|
332
|
+
const { digestEncoding, sortBy = 0, recursive = false, ignore = [], ignoreRoot = [], verbose = true, joinRoot } = options;
|
|
302
333
|
const parent = recursive === 1 && typeof verbose !== 'number';
|
|
303
334
|
let algorithm = options.algorithm;
|
|
304
335
|
from ||= checksumFile(algorithm);
|
|
305
|
-
|
|
306
|
-
switch (algorithm = path.extname(from).substring(1).toLowerCase()) {
|
|
307
|
-
case 'md5':
|
|
308
|
-
case 'sha1':
|
|
309
|
-
case 'sha224':
|
|
310
|
-
case 'sha256':
|
|
311
|
-
case 'sha384':
|
|
312
|
-
case 'sha512':
|
|
313
|
-
case 'ripemd':
|
|
314
|
-
case 'ripemd-160':
|
|
315
|
-
break;
|
|
316
|
-
default:
|
|
317
|
-
algorithm = undefined;
|
|
318
|
-
break;
|
|
319
|
-
}
|
|
320
|
-
}
|
|
336
|
+
algorithm ||= getAlgorithm(asExt(from));
|
|
321
337
|
if (verbose) {
|
|
322
|
-
|
|
338
|
+
setLogCurrent(null);
|
|
323
339
|
}
|
|
324
340
|
let pass = 0, fail = [], missing = [];
|
|
325
341
|
try {
|
|
326
342
|
const filename = path.basename(from);
|
|
327
343
|
const files = [];
|
|
328
344
|
from = joinRoot ? path.join(root, filename) : path.resolve(from);
|
|
329
|
-
recurseDir(files, [root], { ignore:
|
|
345
|
+
recurseDir(files, [root], { ignore: toPosix([...ignore, from]), ignoreRoot: toPosix(ignoreRoot), sortBy, recursive });
|
|
330
346
|
const items = fs.readFileSync(from, 'utf8').split('\n').map(item => {
|
|
331
347
|
const index = item.indexOf(' ');
|
|
332
|
-
return [item.
|
|
348
|
+
return [item.slice(0, index), path.join(root, item.slice(index + 1))];
|
|
333
349
|
});
|
|
334
350
|
const checked = options.include || options.exclude ? filterPaths(root, items.map(item => item[1]), options) : null;
|
|
335
351
|
let valid = false;
|
|
@@ -443,7 +459,7 @@ function setHttpCache(value, type, disk) {
|
|
|
443
459
|
return;
|
|
444
460
|
}
|
|
445
461
|
if (typeof value === 'string') {
|
|
446
|
-
value =
|
|
462
|
+
value = formatSize(value);
|
|
447
463
|
}
|
|
448
464
|
if (value >= 0) {
|
|
449
465
|
switch (type) {
|
|
@@ -480,20 +496,22 @@ function setLogMinWidth() {
|
|
|
480
496
|
}
|
|
481
497
|
const downloadStats = () => [[0, 0], [0, 0], [0, 0]];
|
|
482
498
|
const isCacheable = (item) => item.initialValue?.cacheable !== false;
|
|
483
|
-
const getPmOptions = (value, { matchBase = value.startsWith('*') && !value.includes('/'), dot = false } = {}) => ({ nocase:
|
|
499
|
+
const getPmOptions = (value, { matchBase = value.startsWith('*') && !value.includes('/'), dot = false } = {}) => ({ nocase: Host.PLATFORM_WIN32, matchBase, dot });
|
|
484
500
|
const isDownloadAll = (type) => type === 4 || type === 0;
|
|
485
|
-
const
|
|
501
|
+
const isStored = (noCache, maxAge) => !noCache || typeof maxAge === 'number';
|
|
502
|
+
const hasFiles = (type, uri) => type === 3 || type === 4 || type === 0 && Request.isRclone(uri);
|
|
486
503
|
const hasIncremental = (value) => value === "etag" || value === "exists";
|
|
487
|
-
const
|
|
488
|
-
const
|
|
504
|
+
const hasPath = (values, name) => Host.PLATFORM_WIN32 ? !values.includes(name.toLowerCase()) : !values.includes(name);
|
|
505
|
+
const toPathCase = (value) => Host.PLATFORM_WIN32 ? value.toLowerCase() : value;
|
|
506
|
+
const toPosix = (values) => Host.PLATFORM_WIN32 ? values.map(value => Permission.toPosix(value).toLowerCase()) : values;
|
|
489
507
|
const formatPercent = (value) => Math.round(value).toString().padStart(3) + '%';
|
|
490
508
|
const formatLength = (value, length) => `${length} ${value + (length === 1 ? '' : 's')}`;
|
|
491
|
-
const checkEOF = (value) => value.
|
|
492
|
-
const checksumFile = (algorithm) => "checksum" + '.' + (
|
|
493
|
-
const ignoreAsset = (item, exists) => item.invalid ||
|
|
509
|
+
const checkEOF = (value) => value.at(-1) === '\n' ? value : value + '\n';
|
|
510
|
+
const checksumFile = (algorithm) => "checksum" + '.' + (isString(algorithm) ? algorithm.toLowerCase() : "sha256");
|
|
511
|
+
const ignoreAsset = (item, exists) => item.invalid || hasBit(item.flags, 1 | (!exists ? 128 : 0));
|
|
494
512
|
const validateChecksum = (host, item, buffer, localUri = item.localUri) => !item.checksum || host.checkHash(item.checksum, buffer, localUri);
|
|
495
|
-
const errorChecksum = (algorithm) =>
|
|
496
|
-
const errorAbsolute = (hint) =>
|
|
513
|
+
const errorChecksum = (algorithm) => errorValue("Invalid parameters", algorithm || "sha256");
|
|
514
|
+
const errorAbsolute = (hint) => errorValue("Path is not absolute", hint);
|
|
497
515
|
class Scheduler {
|
|
498
516
|
id = 0;
|
|
499
517
|
count = 0;
|
|
@@ -513,7 +531,7 @@ class Scheduler {
|
|
|
513
531
|
#columns = -1;
|
|
514
532
|
add(id, url) {
|
|
515
533
|
if (url.length > LOGGER.VALUE_WIDTH) {
|
|
516
|
-
url = url.
|
|
534
|
+
url = url.slice(0, LOGGER.VALUE_WIDTH - 3) + '...';
|
|
517
535
|
}
|
|
518
536
|
else {
|
|
519
537
|
url = url.padEnd(LOGGER.VALUE_WIDTH);
|
|
@@ -585,7 +603,7 @@ class Scheduler {
|
|
|
585
603
|
return width >= 10 ? Math.max(Math.min(width, LOGGER.PROGRESS_MAXWIDTH), 10) : -1;
|
|
586
604
|
}
|
|
587
605
|
get rawMode() {
|
|
588
|
-
return
|
|
606
|
+
return Host.PLATFORM_WIN32 && LOGGER.PROGRESS_RAWMODE;
|
|
589
607
|
}
|
|
590
608
|
}
|
|
591
609
|
class TaskStatus {
|
|
@@ -623,20 +641,20 @@ class HttpDiskCache {
|
|
|
623
641
|
if (!this.enabled) {
|
|
624
642
|
return false;
|
|
625
643
|
}
|
|
626
|
-
|
|
627
|
-
|
|
644
|
+
if (typeof uri === 'string') {
|
|
645
|
+
try {
|
|
628
646
|
uri = new URL(uri);
|
|
629
647
|
}
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
648
|
+
catch {
|
|
649
|
+
return false;
|
|
650
|
+
}
|
|
633
651
|
}
|
|
634
652
|
const flags = this._flags;
|
|
635
653
|
if (flags === 0) {
|
|
636
654
|
return true;
|
|
637
655
|
}
|
|
638
656
|
if (flags & (2 | 8)) {
|
|
639
|
-
const ext =
|
|
657
|
+
const ext = asExt(uri.pathname);
|
|
640
658
|
if (ext) {
|
|
641
659
|
if (flags & 2) {
|
|
642
660
|
if (!this._includeExt.some(value => value === ext)) {
|
|
@@ -653,7 +671,14 @@ class HttpDiskCache {
|
|
|
653
671
|
}
|
|
654
672
|
return true;
|
|
655
673
|
}
|
|
656
|
-
add(uri, etag, target, { buffer, contentLength =
|
|
674
|
+
add(uri, etag, target, { buffer, contentLength = getSize(target), maxAge = 0 } = {}) {
|
|
675
|
+
if (maxAge < 0) {
|
|
676
|
+
this.clear(target);
|
|
677
|
+
return;
|
|
678
|
+
}
|
|
679
|
+
if (isNaN(maxAge)) {
|
|
680
|
+
return;
|
|
681
|
+
}
|
|
657
682
|
let tempDir;
|
|
658
683
|
if (contentLength <= this.limit && (tempDir = this.host.getCacheDir(uri))) {
|
|
659
684
|
const baseDir = path.join(tempDir, etag);
|
|
@@ -668,11 +693,11 @@ class HttpDiskCache {
|
|
|
668
693
|
else {
|
|
669
694
|
fs.copyFileSync(target, tempUri);
|
|
670
695
|
}
|
|
671
|
-
const expires = this.expires;
|
|
696
|
+
const expires = maxAge || this.expires;
|
|
672
697
|
if (expires > 0 && expires < Infinity) {
|
|
673
698
|
setTimeout(() => {
|
|
674
699
|
this.clear(target);
|
|
675
|
-
}, Math.min(expires,
|
|
700
|
+
}, Math.min(expires, Host.MAX_TIMEOUT));
|
|
676
701
|
}
|
|
677
702
|
}
|
|
678
703
|
catch {
|
|
@@ -686,7 +711,7 @@ class HttpDiskCache {
|
|
|
686
711
|
return value > 0 && value <= this.limit;
|
|
687
712
|
}
|
|
688
713
|
clear(pathname) {
|
|
689
|
-
|
|
714
|
+
Host.removeDir(path.dirname(pathname));
|
|
690
715
|
}
|
|
691
716
|
set enabled(value) {
|
|
692
717
|
this.#enabled = value;
|
|
@@ -697,9 +722,9 @@ class HttpDiskCache {
|
|
|
697
722
|
set include(value) {
|
|
698
723
|
this._include = [];
|
|
699
724
|
this._includeExt = [];
|
|
700
|
-
if (
|
|
725
|
+
if (isArray(value)) {
|
|
701
726
|
for (const item of value) {
|
|
702
|
-
if (
|
|
727
|
+
if (Host.isURL(item)) {
|
|
703
728
|
this._include.push(item);
|
|
704
729
|
this._flags |= 1;
|
|
705
730
|
}
|
|
@@ -719,9 +744,9 @@ class HttpDiskCache {
|
|
|
719
744
|
set exclude(value) {
|
|
720
745
|
this._exclude = [];
|
|
721
746
|
this._excludeExt = [];
|
|
722
|
-
if (
|
|
747
|
+
if (isArray(value)) {
|
|
723
748
|
for (const item of value) {
|
|
724
|
-
if (
|
|
749
|
+
if (Host.isURL(item)) {
|
|
725
750
|
this._exclude.push(item);
|
|
726
751
|
this._flags |= 4;
|
|
727
752
|
}
|
|
@@ -744,7 +769,7 @@ class HttpMemoryCache extends HttpDiskCache {
|
|
|
744
769
|
expires = MEMORY.EXPIRES;
|
|
745
770
|
toDisk = [MEMORY.DISK_MIN, MEMORY.DISK_MAX];
|
|
746
771
|
constructor(host, enabled) {
|
|
747
|
-
super(host, enabled &&
|
|
772
|
+
super(host, enabled && Host.enabled("memory.settings.users", host.username));
|
|
748
773
|
if (MEMORY.INCLUDE.length > 0) {
|
|
749
774
|
this.include = MEMORY.INCLUDE;
|
|
750
775
|
}
|
|
@@ -755,7 +780,14 @@ class HttpMemoryCache extends HttpDiskCache {
|
|
|
755
780
|
has(uri) {
|
|
756
781
|
return this.expires > 0 && super.has(uri);
|
|
757
782
|
}
|
|
758
|
-
add(uri, etag, buffer, { encoding, toDisk, contentLength = Buffer.byteLength(buffer, encoding) } = {}) {
|
|
783
|
+
add(uri, etag, buffer, { encoding, toDisk, contentLength = Buffer.byteLength(buffer, encoding), maxAge = 0 } = {}) {
|
|
784
|
+
if (maxAge < 0) {
|
|
785
|
+
this.clear(uri);
|
|
786
|
+
return;
|
|
787
|
+
}
|
|
788
|
+
if (isNaN(maxAge)) {
|
|
789
|
+
return;
|
|
790
|
+
}
|
|
759
791
|
if (contentLength <= this.limit) {
|
|
760
792
|
if (uri instanceof URL) {
|
|
761
793
|
uri = uri.href;
|
|
@@ -767,10 +799,11 @@ class HttpMemoryCache extends HttpDiskCache {
|
|
|
767
799
|
const item = [Date.now(), buffer, encoding, contentLength, uri, etag];
|
|
768
800
|
this._items.push(item);
|
|
769
801
|
MEMORY.CACHE.set(uri, item);
|
|
770
|
-
|
|
802
|
+
const expires = maxAge || this.expires;
|
|
803
|
+
if (expires < Infinity) {
|
|
771
804
|
setTimeout(() => {
|
|
772
805
|
this.clear(uri);
|
|
773
|
-
}, Math.min(
|
|
806
|
+
}, Math.min(expires, Host.MAX_TIMEOUT));
|
|
774
807
|
}
|
|
775
808
|
}
|
|
776
809
|
else {
|
|
@@ -846,15 +879,15 @@ class ProcessFile {
|
|
|
846
879
|
createFolder(pathname, emptyDir) {
|
|
847
880
|
const emptied = this.groupData.emptied;
|
|
848
881
|
if (!emptied.includes(pathname)) {
|
|
849
|
-
if (emptyDir &&
|
|
882
|
+
if (emptyDir && Host.removeDir(pathname, true)) {
|
|
850
883
|
emptied.push(pathname);
|
|
851
884
|
}
|
|
852
|
-
else if (
|
|
885
|
+
else if (Host.createDir(pathname)) {
|
|
853
886
|
emptied.push(pathname);
|
|
854
887
|
}
|
|
855
888
|
else {
|
|
856
889
|
this.file.invalid = true;
|
|
857
|
-
this.host.writeFail("Unable to create directory",
|
|
890
|
+
this.host.writeFail("Unable to create directory", errorValue("Path not found", pathname));
|
|
858
891
|
return false;
|
|
859
892
|
}
|
|
860
893
|
}
|
|
@@ -866,7 +899,7 @@ class ProcessFile {
|
|
|
866
899
|
}
|
|
867
900
|
const { host, groupData: group, file, localUri } = this;
|
|
868
901
|
const { bundleId, bundleIndex = -1 } = file;
|
|
869
|
-
if (!
|
|
902
|
+
if (!isEmpty(bundleId) && bundleIndex >= 0) {
|
|
870
903
|
const items = group.appending[localUri] ||= [];
|
|
871
904
|
group.bundling[file.uri] ||= [];
|
|
872
905
|
if (bundleIndex > 0) {
|
|
@@ -876,10 +909,14 @@ class ProcessFile {
|
|
|
876
909
|
const parent = host.assets.find(item => item.bundleIndex === 0 && item.bundleId === bundleId);
|
|
877
910
|
if (parent) {
|
|
878
911
|
(parent.bundleQueue ||= []).push(new Promise(resolve => {
|
|
879
|
-
host.Request.open(url, { method: 'HEAD', httpVersion: 1 })
|
|
912
|
+
host.Request.open(url, { method: 'HEAD', httpVersion: 1, headers: file.headers })
|
|
880
913
|
.on('response', res => {
|
|
881
|
-
|
|
882
|
-
|
|
914
|
+
const statusCode = res.statusCode;
|
|
915
|
+
if (statusCode === 204 || statusCode === 304) {
|
|
916
|
+
file.contentLength = 0;
|
|
917
|
+
}
|
|
918
|
+
else if (statusCode < 300) {
|
|
919
|
+
processHeaders(res.headers, file, false, false);
|
|
883
920
|
}
|
|
884
921
|
resolve(file);
|
|
885
922
|
})
|
|
@@ -925,7 +962,7 @@ class ProcessFile {
|
|
|
925
962
|
file.invalid = true;
|
|
926
963
|
host.filesToRemove.add(localUri);
|
|
927
964
|
host.completeAsyncTask();
|
|
928
|
-
host.writeFail(["Checksum did not match", path.basename(localUri)],
|
|
965
|
+
host.writeFail(["Checksum did not match", path.basename(localUri)], errorValue(file.uri || localUri, "Invalid checksum"), { type: 32, queue: true });
|
|
929
966
|
}
|
|
930
967
|
else {
|
|
931
968
|
if (fetched) {
|
|
@@ -951,7 +988,7 @@ class ProcessFile {
|
|
|
951
988
|
let items = uriMap.get(destUri);
|
|
952
989
|
if (!items) {
|
|
953
990
|
const pathname = path.dirname(destUri);
|
|
954
|
-
if (!
|
|
991
|
+
if (!Host.createDir(pathname)) {
|
|
955
992
|
file.invalid = true;
|
|
956
993
|
continue;
|
|
957
994
|
}
|
|
@@ -984,7 +1021,7 @@ class ProcessFile {
|
|
|
984
1021
|
}
|
|
985
1022
|
async finalize(incremental) {
|
|
986
1023
|
const { host, file, localUri, groupData } = this;
|
|
987
|
-
const bundleStart = file.bundleIndex === 0 && !
|
|
1024
|
+
const bundleStart = file.bundleIndex === 0 && !isEmpty(file.bundleId) ? host.incremental !== "staging" && host.retryLimit === RETRY_LIMIT ? 2 : 1 : 0;
|
|
988
1025
|
groupData.completed[localUri] = file;
|
|
989
1026
|
if (bundleStart > 0) {
|
|
990
1027
|
host.setAssetContent(file, host.getUTF8String(file, localUri));
|
|
@@ -1006,11 +1043,11 @@ class ProcessFile {
|
|
|
1006
1043
|
for (const queue of items) {
|
|
1007
1044
|
const encoding = queue.encoding ||= 'utf8';
|
|
1008
1045
|
const { uri, fetchType: type } = queue;
|
|
1009
|
-
let tempFile;
|
|
1046
|
+
let tempFile, maxAge;
|
|
1010
1047
|
if (checkEtag && queue.trailingContent) {
|
|
1011
1048
|
checkEtag = false;
|
|
1012
1049
|
}
|
|
1013
|
-
const verifyBundle = (value, etag, checked) => {
|
|
1050
|
+
const verifyBundle = (value, etag, checked, noCache) => {
|
|
1014
1051
|
if (!queue.invalid) {
|
|
1015
1052
|
if (!checked && !validateChecksum(host, queue, value, localUri)) {
|
|
1016
1053
|
queue.invalid = true;
|
|
@@ -1020,10 +1057,11 @@ class ProcessFile {
|
|
|
1020
1057
|
value = value.toString(encoding);
|
|
1021
1058
|
}
|
|
1022
1059
|
const url = queue.url;
|
|
1023
|
-
if (etag && host.cacheToMemory.has(url)) {
|
|
1060
|
+
if (etag && host.cacheToMemory.has(url) && isStored(noCache, maxAge)) {
|
|
1024
1061
|
host.cacheToMemory.add(url, encodeURIComponent(etag), value, {
|
|
1025
|
-
encoding,
|
|
1026
1062
|
contentLength: queue.contentLength,
|
|
1063
|
+
maxAge,
|
|
1064
|
+
encoding,
|
|
1027
1065
|
toDisk: !tempFile && isCacheable(queue) ? queue.filename || queue.localUri && path.basename(queue.localUri) : ''
|
|
1028
1066
|
});
|
|
1029
1067
|
}
|
|
@@ -1040,6 +1078,8 @@ class ProcessFile {
|
|
|
1040
1078
|
const options = {
|
|
1041
1079
|
url,
|
|
1042
1080
|
encoding,
|
|
1081
|
+
headers: queue.headers,
|
|
1082
|
+
noCache: true,
|
|
1043
1083
|
statusMessage: uri + ` (${queue.bundleIndex})`
|
|
1044
1084
|
};
|
|
1045
1085
|
let etag, pipeTo;
|
|
@@ -1057,7 +1097,7 @@ class ProcessFile {
|
|
|
1057
1097
|
const source = cached[1];
|
|
1058
1098
|
if (validateChecksum(host, queue, source, localUri)) {
|
|
1059
1099
|
verifyBundle(typeof source === 'string' ? source : source.toString(encoding), '', true);
|
|
1060
|
-
host.addDownload(Buffer.byteLength(source, encoding),
|
|
1100
|
+
host.addDownload(Buffer.byteLength(source, encoding), DOWNLOAD_TYPE.CACHE);
|
|
1061
1101
|
continue;
|
|
1062
1102
|
}
|
|
1063
1103
|
}
|
|
@@ -1069,18 +1109,18 @@ class ProcessFile {
|
|
|
1069
1109
|
const baseDir = path.join(host.getCacheDir(url), etagDir);
|
|
1070
1110
|
pipeTo = path.join(baseDir, path.basename(localUri));
|
|
1071
1111
|
try {
|
|
1072
|
-
if (valid &&
|
|
1112
|
+
if (valid && hasSize(pipeTo)) {
|
|
1073
1113
|
const buffer = fs.readFileSync(pipeTo, { encoding });
|
|
1074
1114
|
if (validateChecksum(host, queue, localUri, buffer)) {
|
|
1075
1115
|
verifyBundle(buffer, etag, true);
|
|
1076
|
-
host.addDownload(Buffer.byteLength(buffer, encoding),
|
|
1116
|
+
host.addDownload(Buffer.byteLength(buffer, encoding), DOWNLOAD_TYPE.CACHE);
|
|
1077
1117
|
continue;
|
|
1078
1118
|
}
|
|
1079
1119
|
}
|
|
1080
1120
|
if (!fs.existsSync(baseDir)) {
|
|
1081
1121
|
fs.mkdirSync(baseDir);
|
|
1082
1122
|
}
|
|
1083
|
-
tempFile = path.join(baseDir,
|
|
1123
|
+
tempFile = path.join(baseDir, incrementUUID());
|
|
1084
1124
|
options.pipeTo = tempFile;
|
|
1085
1125
|
}
|
|
1086
1126
|
catch {
|
|
@@ -1092,20 +1132,21 @@ class ProcessFile {
|
|
|
1092
1132
|
options.httpVersion = 1;
|
|
1093
1133
|
}
|
|
1094
1134
|
options.connected = (headers) => {
|
|
1095
|
-
etag = processHeaders(
|
|
1135
|
+
etag = processHeaders(headers, queue, !!host.Watch, false);
|
|
1136
|
+
maxAge = processCacheHeaders(headers, options.noCache, options.noStore);
|
|
1096
1137
|
return true;
|
|
1097
1138
|
};
|
|
1098
1139
|
}
|
|
1099
1140
|
tasks.push(new Promise((resolve, reject) => {
|
|
1100
1141
|
void host.scheduleTask(url || uri, options, (data) => {
|
|
1101
1142
|
if (data) {
|
|
1102
|
-
verifyBundle(data, etag);
|
|
1143
|
+
verifyBundle(data, etag, false, options.noCache);
|
|
1103
1144
|
}
|
|
1104
1145
|
else {
|
|
1105
1146
|
queue.invalid = true;
|
|
1106
1147
|
}
|
|
1107
1148
|
const downloaded = groupData.bundling[uri];
|
|
1108
|
-
if (
|
|
1149
|
+
if (isArray(downloaded)) {
|
|
1109
1150
|
if (data && !queue.invalid) {
|
|
1110
1151
|
if (typeof data === 'string') {
|
|
1111
1152
|
queue.sourceUTF8 = data;
|
|
@@ -1121,7 +1162,7 @@ class ProcessFile {
|
|
|
1121
1162
|
}
|
|
1122
1163
|
}
|
|
1123
1164
|
}
|
|
1124
|
-
if (tempFile && (!pipeTo ||
|
|
1165
|
+
if (tempFile && (!pipeTo || Host.isPath(pipeTo) || !Host.renameFile(tempFile, pipeTo, false))) {
|
|
1125
1166
|
queueMicrotask(() => fs.unlink(tempFile, () => { }));
|
|
1126
1167
|
}
|
|
1127
1168
|
resolve();
|
|
@@ -1138,12 +1179,12 @@ class ProcessFile {
|
|
|
1138
1179
|
const mimeType = queue.mimeType || file.mimeType;
|
|
1139
1180
|
if (!mimeType) {
|
|
1140
1181
|
queue.invalid = true;
|
|
1141
|
-
tasks.push(Promise.reject(
|
|
1182
|
+
tasks.push(Promise.reject(errorValue("MIME not found", uri)));
|
|
1142
1183
|
break;
|
|
1143
1184
|
}
|
|
1144
|
-
const pathname =
|
|
1185
|
+
const pathname = getTempDir(true);
|
|
1145
1186
|
tasks.push(new Promise((resolve, reject) => {
|
|
1146
|
-
void host.scheduleTask(queue.url || uri, { pathname, binOpts: queue.binOpts || file.binOpts }, (result) => {
|
|
1187
|
+
void host.scheduleTask(queue.url || uri, { pathname, binOpts: queue.binOpts || file.binOpts, headers: queue.headers }, (result) => {
|
|
1147
1188
|
if (result.length > 0) {
|
|
1148
1189
|
verifyBundle(bundleTorrent(host, result, mimeType, encoding));
|
|
1149
1190
|
}
|
|
@@ -1161,7 +1202,7 @@ class ProcessFile {
|
|
|
1161
1202
|
tasks.push(fs.promises.readFile(uri, encoding)
|
|
1162
1203
|
.then(data => {
|
|
1163
1204
|
verifyBundle(data);
|
|
1164
|
-
host.addDownload(Buffer.byteLength(data, encoding),
|
|
1205
|
+
host.addDownload(Buffer.byteLength(data, encoding), DOWNLOAD_TYPE.DISK);
|
|
1165
1206
|
})
|
|
1166
1207
|
.catch(() => {
|
|
1167
1208
|
queue.invalid = true;
|
|
@@ -1181,7 +1222,7 @@ class ProcessFile {
|
|
|
1181
1222
|
return false;
|
|
1182
1223
|
});
|
|
1183
1224
|
}
|
|
1184
|
-
else if (checkEtag &&
|
|
1225
|
+
else if (checkEtag && Host.isPath(localUri)) {
|
|
1185
1226
|
host.completeAsyncTask();
|
|
1186
1227
|
groupData.appending[localUri] = undefined;
|
|
1187
1228
|
return;
|
|
@@ -1209,7 +1250,7 @@ class ProcessFile {
|
|
|
1209
1250
|
const uri = file.uri;
|
|
1210
1251
|
const processed = processing[localUri];
|
|
1211
1252
|
const downloaded = downloading[uri];
|
|
1212
|
-
if (
|
|
1253
|
+
if (isArray(downloaded)) {
|
|
1213
1254
|
this.copyDownload(file, downloaded);
|
|
1214
1255
|
}
|
|
1215
1256
|
if (processed) {
|
|
@@ -1242,14 +1283,14 @@ class ProcessFile {
|
|
|
1242
1283
|
if (!preceding) {
|
|
1243
1284
|
host.completeAsyncTask();
|
|
1244
1285
|
}
|
|
1245
|
-
host.writeFail(["Unable to download file", uri], err, err
|
|
1286
|
+
host.writeFail(["Unable to download file", uri], err, isError(err) && err.message.startsWith("Timeout was exceeded") ? 16384 : 1024);
|
|
1246
1287
|
}
|
|
1247
1288
|
}
|
|
1248
|
-
class FileThread extends
|
|
1289
|
+
class FileThread extends AbortComponent {
|
|
1249
1290
|
file;
|
|
1250
1291
|
threadCount;
|
|
1251
|
-
mimeType = '';
|
|
1252
1292
|
startTime = Date.now();
|
|
1293
|
+
#mimeType = '';
|
|
1253
1294
|
#host;
|
|
1254
1295
|
#queuedTasks = [];
|
|
1255
1296
|
constructor(host, file, threadCount) {
|
|
@@ -1278,8 +1319,16 @@ class FileThread extends core_1.AbortComponent {
|
|
|
1278
1319
|
get localUri() {
|
|
1279
1320
|
return this.file.localUri;
|
|
1280
1321
|
}
|
|
1322
|
+
set mimeType(value) {
|
|
1323
|
+
if (typeof value === 'string') {
|
|
1324
|
+
this.#mimeType = value;
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1327
|
+
get mimeType() {
|
|
1328
|
+
return this.#mimeType;
|
|
1329
|
+
}
|
|
1281
1330
|
}
|
|
1282
|
-
class FileManager extends
|
|
1331
|
+
class FileManager extends Host {
|
|
1283
1332
|
static [kFileManager] = true;
|
|
1284
1333
|
static async purgeMemory(percent = 1, limit = 0, parent) {
|
|
1285
1334
|
if (typeof limit === 'boolean') {
|
|
@@ -1292,6 +1341,7 @@ class FileManager extends core_1.Host {
|
|
|
1292
1341
|
result = MEMORY.CACHE.size;
|
|
1293
1342
|
MEMORY.CACHE.clear();
|
|
1294
1343
|
MEMORY.SIZE = 0;
|
|
1344
|
+
CACHE_ETAG.clear();
|
|
1295
1345
|
}
|
|
1296
1346
|
else if (percent > 0) {
|
|
1297
1347
|
const items = Array.from(MEMORY.CACHE.entries());
|
|
@@ -1319,7 +1369,7 @@ class FileManager extends core_1.Host {
|
|
|
1319
1369
|
return false;
|
|
1320
1370
|
}
|
|
1321
1371
|
const { process: proc, memory, download, request, error, logger } = settings;
|
|
1322
|
-
const sub_limit =
|
|
1372
|
+
const sub_limit = asInt(proc?.thread?.sub_limit);
|
|
1323
1373
|
if (sub_limit > 0) {
|
|
1324
1374
|
PROCESS_SUB_LIMIT = sub_limit;
|
|
1325
1375
|
}
|
|
@@ -1329,7 +1379,7 @@ class FileManager extends core_1.Host {
|
|
|
1329
1379
|
}
|
|
1330
1380
|
if (request) {
|
|
1331
1381
|
let { timeout, disk, buffer, connect } = request;
|
|
1332
|
-
if ((timeout =
|
|
1382
|
+
if ((timeout = fromSeconds(timeout)) >= 0) {
|
|
1333
1383
|
HTTP_CLIENT.timeout = timeout;
|
|
1334
1384
|
}
|
|
1335
1385
|
if (disk) {
|
|
@@ -1341,10 +1391,10 @@ class FileManager extends core_1.Host {
|
|
|
1341
1391
|
if (connect) {
|
|
1342
1392
|
this.defineHttpConnect(connect);
|
|
1343
1393
|
}
|
|
1344
|
-
|
|
1394
|
+
Request.loadSettings({ process: settings.process, request, download }, password);
|
|
1345
1395
|
}
|
|
1346
|
-
if (error) {
|
|
1347
|
-
const limit =
|
|
1396
|
+
if (error?.retry_limit) {
|
|
1397
|
+
const limit = asInt(error.retry_limit);
|
|
1348
1398
|
if (limit >= 0 && limit < Infinity) {
|
|
1349
1399
|
RETRY_LIMIT = limit;
|
|
1350
1400
|
}
|
|
@@ -1362,13 +1412,13 @@ class FileManager extends core_1.Host {
|
|
|
1362
1412
|
}
|
|
1363
1413
|
if (logger.progress) {
|
|
1364
1414
|
let { scroll_buffer, max_width, use_color, text_wrap, color, bg_color, raw_mode, box_char } = logger.progress;
|
|
1365
|
-
if ((scroll_buffer =
|
|
1415
|
+
if ((scroll_buffer = asInt(scroll_buffer)) >= 0 && scroll_buffer <= 16) {
|
|
1366
1416
|
LOGGER.PROGRESS_SCROLLBUFFER = scroll_buffer;
|
|
1367
1417
|
}
|
|
1368
1418
|
if (max_width === 'auto' || max_width === '100%') {
|
|
1369
1419
|
LOGGER.PROGRESS_MAXWIDTH = Infinity;
|
|
1370
1420
|
}
|
|
1371
|
-
else if ((max_width =
|
|
1421
|
+
else if ((max_width = asInt(max_width)) >= 10) {
|
|
1372
1422
|
LOGGER.PROGRESS_MAXWIDTH = max_width;
|
|
1373
1423
|
}
|
|
1374
1424
|
if (typeof use_color === 'boolean') {
|
|
@@ -1382,18 +1432,18 @@ class FileManager extends core_1.Host {
|
|
|
1382
1432
|
LOGGER.PROGRESS_TEXTWRAP = text_wrap;
|
|
1383
1433
|
break;
|
|
1384
1434
|
}
|
|
1385
|
-
if (
|
|
1435
|
+
if (isString(color)) {
|
|
1386
1436
|
LOGGER.PROGRESS_COLOR = color;
|
|
1387
1437
|
}
|
|
1388
|
-
if (
|
|
1438
|
+
if (isString(bg_color)) {
|
|
1389
1439
|
LOGGER.PROGRESS_BGCOLOR = bg_color;
|
|
1390
1440
|
}
|
|
1391
|
-
if (
|
|
1441
|
+
if (isString(box_char) && box_char.length === 1) {
|
|
1392
1442
|
LOGGER.PROGRESS_BOXCHAR = box_char;
|
|
1393
1443
|
}
|
|
1394
1444
|
if (typeof raw_mode === 'boolean') {
|
|
1395
1445
|
LOGGER.PROGRESS_RAWMODE = raw_mode;
|
|
1396
|
-
if (raw_mode &&
|
|
1446
|
+
if (raw_mode && Host.PLATFORM_WIN32) {
|
|
1397
1447
|
require('readline').emitKeypressEvents(PROCESS_STDIN);
|
|
1398
1448
|
}
|
|
1399
1449
|
}
|
|
@@ -1415,7 +1465,7 @@ class FileManager extends core_1.Host {
|
|
|
1415
1465
|
}
|
|
1416
1466
|
static sanitizeAssets(assets, exclusions = []) {
|
|
1417
1467
|
for (const item of assets) {
|
|
1418
|
-
if (
|
|
1468
|
+
if (ignoreFlag(item)) {
|
|
1419
1469
|
continue;
|
|
1420
1470
|
}
|
|
1421
1471
|
const initialValue = item.initialValue;
|
|
@@ -1431,7 +1481,7 @@ class FileManager extends core_1.Host {
|
|
|
1431
1481
|
item.flags &= ~(128 | 16);
|
|
1432
1482
|
break;
|
|
1433
1483
|
case 'watch':
|
|
1434
|
-
if (
|
|
1484
|
+
if (isObject(item.watch) && item.watch.assets) {
|
|
1435
1485
|
delete item.watch.assets;
|
|
1436
1486
|
}
|
|
1437
1487
|
case 'buffer':
|
|
@@ -1456,26 +1506,26 @@ class FileManager extends core_1.Host {
|
|
|
1456
1506
|
return assets;
|
|
1457
1507
|
}
|
|
1458
1508
|
static async writeChecksum(root, to, options) {
|
|
1459
|
-
if (
|
|
1509
|
+
if (isObject(to)) {
|
|
1460
1510
|
options = to;
|
|
1461
1511
|
to = undefined;
|
|
1462
1512
|
}
|
|
1463
1513
|
else {
|
|
1464
1514
|
options ||= {};
|
|
1465
1515
|
}
|
|
1466
|
-
const { algorithm, digestEncoding
|
|
1516
|
+
const { algorithm, digestEncoding, sortBy = 0, recursive = false, ignore = [], ignoreRoot = [], verbose = false, joinRoot } = options;
|
|
1467
1517
|
to ||= checksumFile(algorithm);
|
|
1468
1518
|
let result = [];
|
|
1469
1519
|
try {
|
|
1470
1520
|
const filename = path.basename(to);
|
|
1471
1521
|
to = joinRoot ? path.join(root, to) : path.resolve(to);
|
|
1472
|
-
recurseDir(result, [root], { ignore:
|
|
1522
|
+
recurseDir(result, [root], { ignore: toPosix([...ignore, to]), ignoreRoot: toPosix(ignoreRoot), sortBy, recursive });
|
|
1473
1523
|
const output = [];
|
|
1474
1524
|
for (const pathname of result = filterPaths(root, result, options)) {
|
|
1475
1525
|
if (recursive === 1 && path.basename(pathname) === filename) {
|
|
1476
1526
|
continue;
|
|
1477
1527
|
}
|
|
1478
|
-
const current = await this.readHash(pathname, { algorithm, digestEncoding }) + ' ' +
|
|
1528
|
+
const current = await this.readHash(pathname, { algorithm, digestEncoding }) + ' ' + Permission.toPosix(pathname).slice(root.length).replace(/^\//, '');
|
|
1479
1529
|
if (verbose) {
|
|
1480
1530
|
process.stdout.write(current + '\n');
|
|
1481
1531
|
}
|
|
@@ -1504,7 +1554,7 @@ class FileManager extends core_1.Host {
|
|
|
1504
1554
|
return result;
|
|
1505
1555
|
}
|
|
1506
1556
|
static async verifyChecksum(root, from, options) {
|
|
1507
|
-
if (
|
|
1557
|
+
if (isObject(from)) {
|
|
1508
1558
|
options = from;
|
|
1509
1559
|
from = undefined;
|
|
1510
1560
|
}
|
|
@@ -1521,9 +1571,9 @@ class FileManager extends core_1.Host {
|
|
|
1521
1571
|
if (typeof enabled === 'boolean') {
|
|
1522
1572
|
cache.ENABLED = enabled;
|
|
1523
1573
|
}
|
|
1524
|
-
cache.EXPIRES = expires ?
|
|
1525
|
-
cache.INCLUDE =
|
|
1526
|
-
cache.EXCLUDE =
|
|
1574
|
+
cache.EXPIRES = expires ? parseExpires(expires) : 0;
|
|
1575
|
+
cache.INCLUDE = isArray(include) ? include.slice(0) : [];
|
|
1576
|
+
cache.EXCLUDE = isArray(exclude) ? exclude.slice(0) : [];
|
|
1527
1577
|
setHttpCache(limit, 1, disk);
|
|
1528
1578
|
if (disk) {
|
|
1529
1579
|
return;
|
|
@@ -1537,24 +1587,24 @@ class FileManager extends core_1.Host {
|
|
|
1537
1587
|
setHttpCache(to_disk, 3, disk);
|
|
1538
1588
|
setHttpCache(limit_disk, 4, disk);
|
|
1539
1589
|
}
|
|
1540
|
-
if ((purge_amount =
|
|
1590
|
+
if ((purge_amount = asFloat(purge_amount)) > 0) {
|
|
1541
1591
|
MEMORY.PURGE = Math.min(purge_amount, 1);
|
|
1542
1592
|
}
|
|
1543
1593
|
}
|
|
1544
1594
|
static defineHttpConnect({ timeout, retry_wait, retry_after, retry_limit, redirect_limit }) {
|
|
1545
|
-
if ((timeout =
|
|
1595
|
+
if ((timeout = fromSeconds(timeout)) >= 0) {
|
|
1546
1596
|
HTTP_CLIENT.connectTimeout = timeout;
|
|
1547
1597
|
}
|
|
1548
|
-
if ((retry_wait =
|
|
1598
|
+
if ((retry_wait = fromSeconds(retry_wait)) >= 0) {
|
|
1549
1599
|
HTTP_CLIENT.retryWait = Math.min(retry_wait, 600 * 1000);
|
|
1550
1600
|
}
|
|
1551
|
-
if ((retry_after =
|
|
1601
|
+
if ((retry_after = fromSeconds(retry_after)) >= 0) {
|
|
1552
1602
|
HTTP_CLIENT.retryAfter = Math.min(retry_after, this.MAX_TIMEOUT);
|
|
1553
1603
|
}
|
|
1554
|
-
if ((retry_limit =
|
|
1604
|
+
if ((retry_limit = asInt(retry_limit)) >= 0) {
|
|
1555
1605
|
HTTP_CLIENT.retryLimit = retry_limit;
|
|
1556
1606
|
}
|
|
1557
|
-
if ((redirect_limit =
|
|
1607
|
+
if ((redirect_limit = asInt(redirect_limit)) >= 0) {
|
|
1558
1608
|
HTTP_CLIENT.redirectLimit = redirect_limit;
|
|
1559
1609
|
}
|
|
1560
1610
|
}
|
|
@@ -1603,15 +1653,15 @@ class FileManager extends core_1.Host {
|
|
|
1603
1653
|
#baseDirectory;
|
|
1604
1654
|
constructor(baseDirectory, config, permission, postFinalize) {
|
|
1605
1655
|
super(config);
|
|
1606
|
-
if (
|
|
1656
|
+
if (isFunction(permission)) {
|
|
1607
1657
|
postFinalize = permission;
|
|
1608
1658
|
permission = undefined;
|
|
1609
1659
|
}
|
|
1610
1660
|
if (LOG_TIMEPROCESS) {
|
|
1611
|
-
|
|
1661
|
+
Host.initCpuUsage(this);
|
|
1612
1662
|
}
|
|
1613
|
-
this.#baseDirectory = path.normalize(
|
|
1614
|
-
this.permission = permission &&
|
|
1663
|
+
this.#baseDirectory = path.normalize(Host.PLATFORM_WIN32 ? baseDirectory.replace(/[\\/]+$/, '') : baseDirectory.replace(/\/+$/, ''));
|
|
1664
|
+
this.permission = permission && Host.isPermission(permission) ? permission : Host.getPermissionFromSettings();
|
|
1615
1665
|
this.sessionId = FileManager.generateSessionId();
|
|
1616
1666
|
this.supports('permission', false, true);
|
|
1617
1667
|
this.supports('sessionId', false, true);
|
|
@@ -1625,7 +1675,7 @@ class FileManager extends core_1.Host {
|
|
|
1625
1675
|
if (item.tasks) {
|
|
1626
1676
|
this.taskAssets.push(item);
|
|
1627
1677
|
}
|
|
1628
|
-
if (
|
|
1678
|
+
if (usingFlag(item)) {
|
|
1629
1679
|
targeted.push(item);
|
|
1630
1680
|
}
|
|
1631
1681
|
switch (item.encoding) {
|
|
@@ -1636,7 +1686,7 @@ class FileManager extends core_1.Host {
|
|
|
1636
1686
|
item.encoding = 'utf8';
|
|
1637
1687
|
break;
|
|
1638
1688
|
default: {
|
|
1639
|
-
const encoding =
|
|
1689
|
+
const encoding = getEncoding(item.encoding);
|
|
1640
1690
|
item.encoding = encoding === 'utf8' ? undefined : encoding;
|
|
1641
1691
|
break;
|
|
1642
1692
|
}
|
|
@@ -1657,7 +1707,7 @@ class FileManager extends core_1.Host {
|
|
|
1657
1707
|
if (config.timeout) {
|
|
1658
1708
|
applyTimeout(this.#processTimeout = Object.create(null), config.timeout);
|
|
1659
1709
|
}
|
|
1660
|
-
if (
|
|
1710
|
+
if (isFunction(postFinalize)) {
|
|
1661
1711
|
this.on('end', postFinalize);
|
|
1662
1712
|
}
|
|
1663
1713
|
this.cacheToDisk = new HttpDiskCache(this, DISK.ENABLED);
|
|
@@ -1665,10 +1715,10 @@ class FileManager extends core_1.Host {
|
|
|
1665
1715
|
if (typeof config.threads === 'number') {
|
|
1666
1716
|
this.setTaskLimit(config.threads);
|
|
1667
1717
|
}
|
|
1668
|
-
if (
|
|
1669
|
-
this.#diffSource = [config.log.showDiff.map(value =>
|
|
1718
|
+
if (isPlainObject(config.log) && isArray(config.log.showDiff) && this.incremental !== "staging") {
|
|
1719
|
+
this.#diffSource = [config.log.showDiff.map(value => hasGlob(value) ? (value = Permission.toPosix(value), pm(value, getPmOptions(value))) : value), {}];
|
|
1670
1720
|
}
|
|
1671
|
-
const request = new
|
|
1721
|
+
const request = new Request();
|
|
1672
1722
|
request.host = this;
|
|
1673
1723
|
this.Request = request;
|
|
1674
1724
|
if (this.aborted) {
|
|
@@ -1686,12 +1736,12 @@ class FileManager extends core_1.Host {
|
|
|
1686
1736
|
yield file;
|
|
1687
1737
|
}
|
|
1688
1738
|
}
|
|
1689
|
-
add(value, parent, type =
|
|
1739
|
+
add(value, parent, type = FILE_TYPE.TRANSFORM) {
|
|
1690
1740
|
const file = this.removeCwd(value);
|
|
1691
1741
|
if (file) {
|
|
1692
1742
|
this.files.add(file);
|
|
1693
1743
|
if (parent) {
|
|
1694
|
-
if (type &
|
|
1744
|
+
if (type & FILE_TYPE.TRANSFORM) {
|
|
1695
1745
|
const transforms = parent.transforms;
|
|
1696
1746
|
if (!transforms) {
|
|
1697
1747
|
parent.transforms = [value];
|
|
@@ -1700,7 +1750,7 @@ class FileManager extends core_1.Host {
|
|
|
1700
1750
|
transforms.push(value);
|
|
1701
1751
|
}
|
|
1702
1752
|
}
|
|
1703
|
-
if (type &
|
|
1753
|
+
if (type & FILE_TYPE.TORRENT) {
|
|
1704
1754
|
const torrent = parent.torrentFiles;
|
|
1705
1755
|
if (!torrent) {
|
|
1706
1756
|
parent.torrentFiles = [value];
|
|
@@ -1709,7 +1759,7 @@ class FileManager extends core_1.Host {
|
|
|
1709
1759
|
torrent.push(value);
|
|
1710
1760
|
}
|
|
1711
1761
|
}
|
|
1712
|
-
if (type & (
|
|
1762
|
+
if (type & (FILE_TYPE.COMPRESSED | FILE_TYPE.SOURCEMAP)) {
|
|
1713
1763
|
const connected = parent.descendants;
|
|
1714
1764
|
if (!connected) {
|
|
1715
1765
|
parent.descendants = [value];
|
|
@@ -1730,7 +1780,7 @@ class FileManager extends core_1.Host {
|
|
|
1730
1780
|
if (this.files.delete(this.removeCwd(value))) {
|
|
1731
1781
|
if (emptyDir) {
|
|
1732
1782
|
let dir = this.baseDirectory;
|
|
1733
|
-
for (const seg of path.dirname(value).
|
|
1783
|
+
for (const seg of path.dirname(value).slice(this.baseDirectory.length + 1).split(path.sep)) {
|
|
1734
1784
|
if (seg) {
|
|
1735
1785
|
dir += path.sep + seg;
|
|
1736
1786
|
this.emptyDir.add(dir);
|
|
@@ -1757,7 +1807,7 @@ class FileManager extends core_1.Host {
|
|
|
1757
1807
|
this.files.forEach(predicate, thisArg);
|
|
1758
1808
|
}
|
|
1759
1809
|
clear() {
|
|
1760
|
-
|
|
1810
|
+
Host.removeDir(this.baseDirectory, true);
|
|
1761
1811
|
this.files.clear();
|
|
1762
1812
|
}
|
|
1763
1813
|
get size() {
|
|
@@ -1768,11 +1818,11 @@ class FileManager extends core_1.Host {
|
|
|
1768
1818
|
super.abort(reason);
|
|
1769
1819
|
}
|
|
1770
1820
|
willAbort(value) {
|
|
1771
|
-
if (
|
|
1821
|
+
if (isObject(value)) {
|
|
1772
1822
|
value = value.moduleName;
|
|
1773
1823
|
}
|
|
1774
1824
|
const error = this.config.error;
|
|
1775
|
-
return !!error &&
|
|
1825
|
+
return !!error && isArray(error.abort) && error.abort.includes(value);
|
|
1776
1826
|
}
|
|
1777
1827
|
restart(recursive, exclusions, emptyDir) {
|
|
1778
1828
|
if (recursive === 'abort') {
|
|
@@ -1795,7 +1845,7 @@ class FileManager extends core_1.Host {
|
|
|
1795
1845
|
this.processAssets(emptyDir);
|
|
1796
1846
|
}
|
|
1797
1847
|
else {
|
|
1798
|
-
this.writeFail(['Unable to restart session', formatLength('asset', this.assets.length)],
|
|
1848
|
+
this.writeFail(['Unable to restart session', formatLength('asset', this.assets.length)], errorMessage('sessionId', this.sessionId));
|
|
1799
1849
|
}
|
|
1800
1850
|
}
|
|
1801
1851
|
else {
|
|
@@ -1809,8 +1859,8 @@ class FileManager extends core_1.Host {
|
|
|
1809
1859
|
}
|
|
1810
1860
|
switch (name) {
|
|
1811
1861
|
case 'cloud': {
|
|
1812
|
-
if (
|
|
1813
|
-
const instance = new
|
|
1862
|
+
if (isArray(args[1])) {
|
|
1863
|
+
const instance = new Cloud(args[0], args[1]);
|
|
1814
1864
|
instance.host = this;
|
|
1815
1865
|
instance.init(this.config);
|
|
1816
1866
|
this.observeFile(instance);
|
|
@@ -1858,7 +1908,7 @@ class FileManager extends core_1.Host {
|
|
|
1858
1908
|
deleteFile(value, options = {}, callback) {
|
|
1859
1909
|
if (super.deleteFile(value, options, callback)) {
|
|
1860
1910
|
let emptyDir = false;
|
|
1861
|
-
if (
|
|
1911
|
+
if (isObject(options)) {
|
|
1862
1912
|
if (options.emptyDir) {
|
|
1863
1913
|
emptyDir = true;
|
|
1864
1914
|
}
|
|
@@ -1909,7 +1959,7 @@ class FileManager extends core_1.Host {
|
|
|
1909
1959
|
}
|
|
1910
1960
|
case 'task': {
|
|
1911
1961
|
const Module = this.#tryPackage(target, name, 4);
|
|
1912
|
-
if (Module &&
|
|
1962
|
+
if (Module && isPlainObject(args[0])) {
|
|
1913
1963
|
const instance = new Module(...args);
|
|
1914
1964
|
instance.host = this;
|
|
1915
1965
|
instance.init(this.config);
|
|
@@ -1922,11 +1972,11 @@ class FileManager extends core_1.Host {
|
|
|
1922
1972
|
case 'cloud': {
|
|
1923
1973
|
const database = this.dataSourceItems.filter(item => item.source === 'cloud');
|
|
1924
1974
|
let instance;
|
|
1925
|
-
if (
|
|
1975
|
+
if (isString(target)) {
|
|
1926
1976
|
const mod = args.shift();
|
|
1927
|
-
if (
|
|
1977
|
+
if (isObject(mod)) {
|
|
1928
1978
|
if (target === "@e-mc/cloud") {
|
|
1929
|
-
instance = new
|
|
1979
|
+
instance = new Cloud(mod, database);
|
|
1930
1980
|
}
|
|
1931
1981
|
else {
|
|
1932
1982
|
const Module = this.#tryPackage(target, name, 64);
|
|
@@ -1936,8 +1986,8 @@ class FileManager extends core_1.Host {
|
|
|
1936
1986
|
}
|
|
1937
1987
|
}
|
|
1938
1988
|
}
|
|
1939
|
-
else if (
|
|
1940
|
-
instance = new
|
|
1989
|
+
else if (isObject(target)) {
|
|
1990
|
+
instance = new Cloud(target, database);
|
|
1941
1991
|
}
|
|
1942
1992
|
if (!instance) {
|
|
1943
1993
|
return;
|
|
@@ -1949,11 +1999,11 @@ class FileManager extends core_1.Host {
|
|
|
1949
1999
|
}
|
|
1950
2000
|
case 'watch': {
|
|
1951
2001
|
let instance;
|
|
1952
|
-
if (
|
|
2002
|
+
if (isString(target)) {
|
|
1953
2003
|
const mod = args.shift();
|
|
1954
|
-
if (
|
|
2004
|
+
if (isObject(mod)) {
|
|
1955
2005
|
if (target === "@e-mc/watch") {
|
|
1956
|
-
instance = new
|
|
2006
|
+
instance = new Watch(mod);
|
|
1957
2007
|
}
|
|
1958
2008
|
else {
|
|
1959
2009
|
const Module = this.#tryPackage(target, name, 16);
|
|
@@ -1963,8 +2013,8 @@ class FileManager extends core_1.Host {
|
|
|
1963
2013
|
}
|
|
1964
2014
|
}
|
|
1965
2015
|
}
|
|
1966
|
-
else if (
|
|
1967
|
-
instance = new
|
|
2016
|
+
else if (isObject(target)) {
|
|
2017
|
+
instance = new Watch(target);
|
|
1968
2018
|
}
|
|
1969
2019
|
if (!instance) {
|
|
1970
2020
|
return;
|
|
@@ -2048,7 +2098,7 @@ class FileManager extends core_1.Host {
|
|
|
2048
2098
|
this.Image ||= new Map();
|
|
2049
2099
|
}
|
|
2050
2100
|
for (const [mimeType, Module] of target) {
|
|
2051
|
-
if (
|
|
2101
|
+
if (Host.constructorOf(Module, 'image')) {
|
|
2052
2102
|
const instance = new Module(...args);
|
|
2053
2103
|
instance.host = this;
|
|
2054
2104
|
instance.init(this.config);
|
|
@@ -2062,7 +2112,7 @@ class FileManager extends core_1.Host {
|
|
|
2062
2112
|
this.Image = null;
|
|
2063
2113
|
break;
|
|
2064
2114
|
case 'compress': {
|
|
2065
|
-
const instance = new
|
|
2115
|
+
const instance = new Compress(target);
|
|
2066
2116
|
instance.host = this;
|
|
2067
2117
|
instance.init();
|
|
2068
2118
|
this.observeFile(instance);
|
|
@@ -2072,12 +2122,12 @@ class FileManager extends core_1.Host {
|
|
|
2072
2122
|
}
|
|
2073
2123
|
findAsset(value, instance) {
|
|
2074
2124
|
let assets, replaced;
|
|
2075
|
-
if (
|
|
2125
|
+
if (isPlainObject(instance)) {
|
|
2076
2126
|
({ instance, assets, replaced } = instance);
|
|
2077
2127
|
}
|
|
2078
2128
|
let origin, pathname;
|
|
2079
2129
|
if (typeof value === 'string') {
|
|
2080
|
-
if (
|
|
2130
|
+
if (Host.isURL(value, 'file')) {
|
|
2081
2131
|
try {
|
|
2082
2132
|
({ origin, pathname, href: value } = new URL(value));
|
|
2083
2133
|
}
|
|
@@ -2086,18 +2136,18 @@ class FileManager extends core_1.Host {
|
|
|
2086
2136
|
}
|
|
2087
2137
|
}
|
|
2088
2138
|
else {
|
|
2089
|
-
value =
|
|
2139
|
+
value = Host.resolveFile(value);
|
|
2090
2140
|
}
|
|
2091
2141
|
}
|
|
2092
2142
|
else {
|
|
2093
2143
|
({ origin, pathname, href: value } = value);
|
|
2094
2144
|
}
|
|
2095
|
-
if (!
|
|
2145
|
+
if (!isString(value)) {
|
|
2096
2146
|
return;
|
|
2097
2147
|
}
|
|
2098
2148
|
const result = (assets || this.assets).filter(item => {
|
|
2099
2149
|
const { uri, flags } = item;
|
|
2100
|
-
if (uri && (!instance || this.hasDocument(instance, item.document)) && (!item.invalid || assets ||
|
|
2150
|
+
if (uri && (!instance || this.hasDocument(instance, item.document)) && (!item.invalid || assets || mainFlag(flags) || usingFlag(flags) || cloneFlag(flags) || watchFlag(flags))) {
|
|
2101
2151
|
if (origin) {
|
|
2102
2152
|
if (!item.url) {
|
|
2103
2153
|
try {
|
|
@@ -2118,7 +2168,7 @@ class FileManager extends core_1.Host {
|
|
|
2118
2168
|
const indexA = uri.indexOf('#');
|
|
2119
2169
|
const indexB = value.indexOf('#');
|
|
2120
2170
|
if (indexA !== -1 || indexB !== -1) {
|
|
2121
|
-
return equalAddress(indexA !== -1 ? uri.
|
|
2171
|
+
return equalAddress(indexA !== -1 ? uri.slice(0, indexA) : uri, indexB !== -1 ? value.slice(0, indexB) : value);
|
|
2122
2172
|
}
|
|
2123
2173
|
}
|
|
2124
2174
|
return false;
|
|
@@ -2126,7 +2176,7 @@ class FileManager extends core_1.Host {
|
|
|
2126
2176
|
if (replaced) {
|
|
2127
2177
|
const map = this.#replacedAssets;
|
|
2128
2178
|
if (map) {
|
|
2129
|
-
const target = pathname ||
|
|
2179
|
+
const target = pathname || Host.toPosix(value);
|
|
2130
2180
|
const found = [];
|
|
2131
2181
|
for (const relative in map) {
|
|
2132
2182
|
const item = map[relative];
|
|
@@ -2153,12 +2203,12 @@ class FileManager extends core_1.Host {
|
|
|
2153
2203
|
return 1;
|
|
2154
2204
|
}
|
|
2155
2205
|
const bundleId = a.bundleId;
|
|
2156
|
-
if ((
|
|
2206
|
+
if ((isEmpty(bundleId) || a.bundleIndex === 0) && (isEmpty(b.bundleId) || b.bundleIndex === 0)) {
|
|
2157
2207
|
if (a.id && b.id) {
|
|
2158
2208
|
return a.id < b.id ? -1 : 1;
|
|
2159
2209
|
}
|
|
2160
2210
|
}
|
|
2161
|
-
else if (!
|
|
2211
|
+
else if (!isEmpty(bundleId) && bundleId === b.bundleId) {
|
|
2162
2212
|
return a.bundleIndex < b.bundleIndex ? -1 : 1;
|
|
2163
2213
|
}
|
|
2164
2214
|
return 0;
|
|
@@ -2175,16 +2225,16 @@ class FileManager extends core_1.Host {
|
|
|
2175
2225
|
removeCwd(value) {
|
|
2176
2226
|
if (typeof value === 'string') {
|
|
2177
2227
|
const baseDir = this.baseDirectory + path.sep;
|
|
2178
|
-
const leading = value.
|
|
2179
|
-
if (leading === baseDir ||
|
|
2180
|
-
return value.
|
|
2228
|
+
const leading = value.slice(0, baseDir.length);
|
|
2229
|
+
if (leading === baseDir || Host.PLATFORM_WIN32 && leading.toLowerCase() === baseDir.toLowerCase()) {
|
|
2230
|
+
return value.slice(baseDir.length);
|
|
2181
2231
|
}
|
|
2182
2232
|
}
|
|
2183
2233
|
return '';
|
|
2184
2234
|
}
|
|
2185
2235
|
replace(file, replaceWith, mimeType) {
|
|
2186
2236
|
let ignoreExt, rewritePath;
|
|
2187
|
-
if (
|
|
2237
|
+
if (isObject(mimeType)) {
|
|
2188
2238
|
({ mimeType, ignoreExt, rewritePath } = mimeType);
|
|
2189
2239
|
}
|
|
2190
2240
|
const { pathname, filename, localUri } = file;
|
|
@@ -2192,7 +2242,7 @@ class FileManager extends core_1.Host {
|
|
|
2192
2242
|
const name = path.basename(replaceWith);
|
|
2193
2243
|
if (!ignoreExt && localUri && path.extname(localUri) === path.extname(replaceWith)) {
|
|
2194
2244
|
try {
|
|
2195
|
-
if (!
|
|
2245
|
+
if (!Host.renameFile(replaceWith, localUri)) {
|
|
2196
2246
|
this.filesToRemove.add(replaceWith);
|
|
2197
2247
|
}
|
|
2198
2248
|
}
|
|
@@ -2202,7 +2252,7 @@ class FileManager extends core_1.Host {
|
|
|
2202
2252
|
}
|
|
2203
2253
|
}
|
|
2204
2254
|
else {
|
|
2205
|
-
|
|
2255
|
+
setInitialValue(file);
|
|
2206
2256
|
if (rewritePath || !localUri) {
|
|
2207
2257
|
file.pathname = this.removeCwd(path.dirname(replaceWith));
|
|
2208
2258
|
}
|
|
@@ -2210,7 +2260,7 @@ class FileManager extends core_1.Host {
|
|
|
2210
2260
|
const replacePath = this.setLocalUri(file, true).localUri;
|
|
2211
2261
|
if (replaceWith !== replacePath) {
|
|
2212
2262
|
try {
|
|
2213
|
-
if (!
|
|
2263
|
+
if (!Host.renameFile(replaceWith, replacePath)) {
|
|
2214
2264
|
this.filesToRemove.add(replaceWith);
|
|
2215
2265
|
}
|
|
2216
2266
|
}
|
|
@@ -2222,7 +2272,7 @@ class FileManager extends core_1.Host {
|
|
|
2222
2272
|
return false;
|
|
2223
2273
|
}
|
|
2224
2274
|
}
|
|
2225
|
-
file.mimeType = mimeType ||
|
|
2275
|
+
file.mimeType = mimeType || Host.lookupMime(name) || file.mimeType;
|
|
2226
2276
|
this.add(replaceWith);
|
|
2227
2277
|
if (localUri && !this.assets.find(item => item.localUri === localUri && !item.invalid)) {
|
|
2228
2278
|
this.filesToRemove.add(localUri);
|
|
@@ -2248,7 +2298,7 @@ class FileManager extends core_1.Host {
|
|
|
2248
2298
|
if (copy) {
|
|
2249
2299
|
fs.copyFileSync(oldPath, newPath);
|
|
2250
2300
|
}
|
|
2251
|
-
else if (!
|
|
2301
|
+
else if (!Host.renameFile(oldPath, newPath)) {
|
|
2252
2302
|
this.filesToRemove.add(oldPath);
|
|
2253
2303
|
}
|
|
2254
2304
|
}
|
|
@@ -2261,7 +2311,7 @@ class FileManager extends core_1.Host {
|
|
|
2261
2311
|
this.delete(oldPath);
|
|
2262
2312
|
}
|
|
2263
2313
|
}
|
|
2264
|
-
|
|
2314
|
+
setInitialValue(file);
|
|
2265
2315
|
file.filename = value;
|
|
2266
2316
|
this.add(file.localUri = this.setLocalUri(file, true).localUri);
|
|
2267
2317
|
}
|
|
@@ -2291,7 +2341,7 @@ class FileManager extends core_1.Host {
|
|
|
2291
2341
|
this.clearProcessTimeout();
|
|
2292
2342
|
this.#delayed = Infinity;
|
|
2293
2343
|
if (this.aborted) {
|
|
2294
|
-
this.abortFinalize(
|
|
2344
|
+
this.abortFinalize(createAbortError());
|
|
2295
2345
|
return;
|
|
2296
2346
|
}
|
|
2297
2347
|
this.finalizeState = 1;
|
|
@@ -2313,12 +2363,12 @@ class FileManager extends core_1.Host {
|
|
|
2313
2363
|
this.resetState();
|
|
2314
2364
|
const log = this.config.log;
|
|
2315
2365
|
let useNumeric, showSize;
|
|
2316
|
-
if (
|
|
2366
|
+
if (isObject(log)) {
|
|
2317
2367
|
({ useNumeric, showSize } = log);
|
|
2318
2368
|
}
|
|
2319
2369
|
const items = showSize === false ? files : files.map(name => {
|
|
2320
|
-
const size =
|
|
2321
|
-
return { name, size: useNumeric ? size :
|
|
2370
|
+
const size = getSize(path.join(this.baseDirectory, name));
|
|
2371
|
+
return { name, size: useNumeric ? size : formatSize(size) };
|
|
2322
2372
|
});
|
|
2323
2373
|
this.emit('end', items, errors, this.collectLog());
|
|
2324
2374
|
this.done = true;
|
|
@@ -2337,13 +2387,13 @@ class FileManager extends core_1.Host {
|
|
|
2337
2387
|
pathname = file.pathname || '';
|
|
2338
2388
|
}
|
|
2339
2389
|
const assets = this.assets;
|
|
2340
|
-
let target =
|
|
2390
|
+
let target = toPathCase(path.join(pathname, filename)), modified = false;
|
|
2341
2391
|
for (let i = 0, j = 1, length = assets.length; i < length; ++i) {
|
|
2342
2392
|
const item = assets[i];
|
|
2343
|
-
if (item !== file && item.filename && target ===
|
|
2393
|
+
if (item !== file && item.filename && target === toPathCase(path.join(item.pathname || '', item.filename))) {
|
|
2344
2394
|
const { name, ext } = path.parse(filename);
|
|
2345
2395
|
filename = name + '_' + j + ext;
|
|
2346
|
-
target =
|
|
2396
|
+
target = toPathCase(path.join(pathname, filename));
|
|
2347
2397
|
modified = true;
|
|
2348
2398
|
i = -1;
|
|
2349
2399
|
++j;
|
|
@@ -2358,14 +2408,14 @@ class FileManager extends core_1.Host {
|
|
|
2358
2408
|
setLocalUri(file, replace) {
|
|
2359
2409
|
let uri = file.uri || file.url?.toString(), type;
|
|
2360
2410
|
if (uri && !file.content && !file.base64 && !file.dataView) {
|
|
2361
|
-
if (
|
|
2411
|
+
if (Host.isFile(uri, 'torrent')) {
|
|
2362
2412
|
type = 4;
|
|
2363
2413
|
}
|
|
2364
|
-
else if (
|
|
2414
|
+
else if (Request.isRclone(uri)) {
|
|
2365
2415
|
type = 0;
|
|
2366
2416
|
}
|
|
2367
2417
|
else if (!file.url) {
|
|
2368
|
-
if (
|
|
2418
|
+
if (Host.isURL(uri, 'file')) {
|
|
2369
2419
|
try {
|
|
2370
2420
|
file.url = new URL(uri);
|
|
2371
2421
|
file.uri = file.url.toString();
|
|
@@ -2380,7 +2430,7 @@ class FileManager extends core_1.Host {
|
|
|
2380
2430
|
}
|
|
2381
2431
|
else {
|
|
2382
2432
|
try {
|
|
2383
|
-
if (!(file.uri =
|
|
2433
|
+
if (!(file.uri = Host.resolveFile(uri))) {
|
|
2384
2434
|
throw errorAbsolute(uri);
|
|
2385
2435
|
}
|
|
2386
2436
|
if (file.socketPath) {
|
|
@@ -2404,7 +2454,7 @@ class FileManager extends core_1.Host {
|
|
|
2404
2454
|
}
|
|
2405
2455
|
const url = file.url;
|
|
2406
2456
|
if (file.pathname) {
|
|
2407
|
-
file.pathname =
|
|
2457
|
+
file.pathname = Host.toPosix(file.pathname);
|
|
2408
2458
|
}
|
|
2409
2459
|
if (url) {
|
|
2410
2460
|
if (type === undefined) {
|
|
@@ -2428,7 +2478,7 @@ class FileManager extends core_1.Host {
|
|
|
2428
2478
|
if (!filename && !file.document && !isDownloadAll(type)) {
|
|
2429
2479
|
filename = url.pathname.split('/').pop();
|
|
2430
2480
|
if (filename.includes('.')) {
|
|
2431
|
-
const mimeType = file.mimeType ||
|
|
2481
|
+
const mimeType = file.mimeType || Host.lookupMime(filename);
|
|
2432
2482
|
if (mimeType) {
|
|
2433
2483
|
file.filename = filename;
|
|
2434
2484
|
file.mimeType = mimeType;
|
|
@@ -2436,7 +2486,7 @@ class FileManager extends core_1.Host {
|
|
|
2436
2486
|
}
|
|
2437
2487
|
}
|
|
2438
2488
|
else if (file.mimeType && filename) {
|
|
2439
|
-
const ext =
|
|
2489
|
+
const ext = Host.lookupMime(file.mimeType, true);
|
|
2440
2490
|
if (ext) {
|
|
2441
2491
|
filename += '.' + ext;
|
|
2442
2492
|
file.filename = filename;
|
|
@@ -2464,7 +2514,7 @@ class FileManager extends core_1.Host {
|
|
|
2464
2514
|
const localUri = file.filename ? path.join(pathname, file.filename) : pathname;
|
|
2465
2515
|
file.localUri = localUri;
|
|
2466
2516
|
if (!file.mimeType && !isDownloadAll(type)) {
|
|
2467
|
-
file.mimeType = file.url &&
|
|
2517
|
+
file.mimeType = file.url && Host.lookupMime(path.basename(file.url.pathname)) || Host.lookupMime(file.filename);
|
|
2468
2518
|
}
|
|
2469
2519
|
return { pathname, localUri };
|
|
2470
2520
|
}
|
|
@@ -2472,7 +2522,7 @@ class FileManager extends core_1.Host {
|
|
|
2472
2522
|
return data.file.localUri || '';
|
|
2473
2523
|
}
|
|
2474
2524
|
getMimeType(data) {
|
|
2475
|
-
return data.mimeType ||= data.file.mimeType ||
|
|
2525
|
+
return data.mimeType ||= data.file.mimeType || Host.lookupMime(path.basename(this.getLocalUri(data)));
|
|
2476
2526
|
}
|
|
2477
2527
|
getDocumentAssets(instance, condition) {
|
|
2478
2528
|
return this.documentAssets.filter(item => this.hasDocument(instance, item.document) && (!condition || condition(item)));
|
|
@@ -2483,7 +2533,7 @@ class FileManager extends core_1.Host {
|
|
|
2483
2533
|
getUTF8String(file, uri) {
|
|
2484
2534
|
let sourceUTF8 = file.sourceUTF8;
|
|
2485
2535
|
if (!sourceUTF8) {
|
|
2486
|
-
file.encoding =
|
|
2536
|
+
file.encoding = getEncoding(file.encoding);
|
|
2487
2537
|
if (file.buffer) {
|
|
2488
2538
|
sourceUTF8 = file.buffer.toString(file.encoding);
|
|
2489
2539
|
file.buffer = null;
|
|
@@ -2497,7 +2547,7 @@ class FileManager extends core_1.Host {
|
|
|
2497
2547
|
}
|
|
2498
2548
|
}
|
|
2499
2549
|
if (sourceUTF8?.[0] === '\uFEFF' && file.encoding !== 'utf16le') {
|
|
2500
|
-
sourceUTF8 = sourceUTF8.
|
|
2550
|
+
sourceUTF8 = sourceUTF8.slice(1);
|
|
2501
2551
|
}
|
|
2502
2552
|
return file.sourceUTF8 = sourceUTF8 || '';
|
|
2503
2553
|
}
|
|
@@ -2510,13 +2560,13 @@ class FileManager extends core_1.Host {
|
|
|
2510
2560
|
let { base64, localUri } = file;
|
|
2511
2561
|
if (localUri) {
|
|
2512
2562
|
if (typeof minStreamSize === 'number') {
|
|
2513
|
-
return
|
|
2563
|
+
return Host.streamFile(localUri, { minStreamSize, cache: false, signal: this.signal });
|
|
2514
2564
|
}
|
|
2515
2565
|
try {
|
|
2516
2566
|
return fs.readFileSync(localUri);
|
|
2517
2567
|
}
|
|
2518
2568
|
catch (err) {
|
|
2519
|
-
if (!(base64 &&
|
|
2569
|
+
if (!(base64 && isErrorCode(err, 'ENOENT', 'EACCES'))) {
|
|
2520
2570
|
this.writeFail(["Unable to read file", path.basename(localUri)], err, 32);
|
|
2521
2571
|
base64 = undefined;
|
|
2522
2572
|
}
|
|
@@ -2537,7 +2587,7 @@ class FileManager extends core_1.Host {
|
|
|
2537
2587
|
return this.getTempDir({ pathname: url.hostname + '_' + (url.port || (url.protocol === 'https:' ? '443' : '80')), moduleDir: true, createDir });
|
|
2538
2588
|
}
|
|
2539
2589
|
setAssetContent(file, content, options) {
|
|
2540
|
-
let trailing =
|
|
2590
|
+
let trailing = concatString(file.trailingContent);
|
|
2541
2591
|
if (file.document) {
|
|
2542
2592
|
for (const { instance } of this.Document) {
|
|
2543
2593
|
if (instance.resolveUri && this.hasDocument(instance, file.document)) {
|
|
@@ -2565,7 +2615,7 @@ class FileManager extends core_1.Host {
|
|
|
2565
2615
|
this.contentToReplace.set(localUri, replacing = []);
|
|
2566
2616
|
}
|
|
2567
2617
|
appending[bundleIndex - 1] = content;
|
|
2568
|
-
if (
|
|
2618
|
+
if (isString(options.bundleReplace)) {
|
|
2569
2619
|
replacing[bundleIndex - 1] = options.bundleReplace;
|
|
2570
2620
|
}
|
|
2571
2621
|
file.invalid = true;
|
|
@@ -2580,10 +2630,11 @@ class FileManager extends core_1.Host {
|
|
|
2580
2630
|
return content;
|
|
2581
2631
|
}
|
|
2582
2632
|
let replacing, newline;
|
|
2583
|
-
const getSeparator = (value) => newline ||=
|
|
2633
|
+
const getSeparator = (value) => newline ||= getNewline(value);
|
|
2584
2634
|
if (content && (replacing = this.contentToReplace.get(localUri)) && replacing.length > 0) {
|
|
2585
|
-
for (let i = 0,
|
|
2586
|
-
|
|
2635
|
+
for (let i = 0, match; i < replacing.length; ++i) {
|
|
2636
|
+
let value = appending[i];
|
|
2637
|
+
if (isString(value)) {
|
|
2587
2638
|
if (replacing[i] && (match = new RegExp(replacing[i], 'i').exec(content))) {
|
|
2588
2639
|
if (document) {
|
|
2589
2640
|
for (const { instance } of this.Document) {
|
|
@@ -2596,7 +2647,7 @@ class FileManager extends core_1.Host {
|
|
|
2596
2647
|
}
|
|
2597
2648
|
}
|
|
2598
2649
|
}
|
|
2599
|
-
content = content.
|
|
2650
|
+
content = content.slice(0, match.index) + value + getSeparator(content) + content.slice(match.index + match[0].length);
|
|
2600
2651
|
}
|
|
2601
2652
|
else {
|
|
2602
2653
|
content += getSeparator(content) + value;
|
|
@@ -2636,7 +2687,7 @@ class FileManager extends core_1.Host {
|
|
|
2636
2687
|
return;
|
|
2637
2688
|
}
|
|
2638
2689
|
const document = data.file.document;
|
|
2639
|
-
const ext =
|
|
2690
|
+
const ext = asExt(localUri);
|
|
2640
2691
|
let output;
|
|
2641
2692
|
saveAs ||= ext;
|
|
2642
2693
|
if (document) {
|
|
@@ -2651,7 +2702,7 @@ class FileManager extends core_1.Host {
|
|
|
2651
2702
|
if (!replace || this.filesQueued.has(localUri)) {
|
|
2652
2703
|
let i = 1;
|
|
2653
2704
|
do {
|
|
2654
|
-
output =
|
|
2705
|
+
output = renameExt(localUri, '__copy__.' + (i > 1 ? `(${i}).` : '') + saveAs);
|
|
2655
2706
|
} while (this.filesQueued.has(output) && ++i);
|
|
2656
2707
|
try {
|
|
2657
2708
|
fs.copyFileSync(localUri, output);
|
|
@@ -2665,7 +2716,7 @@ class FileManager extends core_1.Host {
|
|
|
2665
2716
|
else {
|
|
2666
2717
|
let i = 1;
|
|
2667
2718
|
do {
|
|
2668
|
-
output =
|
|
2719
|
+
output = renameExt(localUri, (i > 1 ? `(${i}).` : '') + saveAs);
|
|
2669
2720
|
if (replace) {
|
|
2670
2721
|
break;
|
|
2671
2722
|
}
|
|
@@ -2675,7 +2726,7 @@ class FileManager extends core_1.Host {
|
|
|
2675
2726
|
return output;
|
|
2676
2727
|
}
|
|
2677
2728
|
handleFilePermission(file) {
|
|
2678
|
-
this.writeFail(["Unable to read file", file.uri && path.basename(file.uri)],
|
|
2729
|
+
this.writeFail(["Unable to read file", file.uri && path.basename(file.uri)], errorValue("Operation not permitted", file.uri || file.filename), 8192);
|
|
2679
2730
|
file.invalid = true;
|
|
2680
2731
|
this.emit('asset:permission', file);
|
|
2681
2732
|
}
|
|
@@ -2686,7 +2737,7 @@ class FileManager extends core_1.Host {
|
|
|
2686
2737
|
}
|
|
2687
2738
|
let mimeType = '', ext;
|
|
2688
2739
|
try {
|
|
2689
|
-
const result = await
|
|
2740
|
+
const result = await Host.resolveMime(file.buffer || localUri);
|
|
2690
2741
|
if (result) {
|
|
2691
2742
|
({ mime: mimeType, ext } = result);
|
|
2692
2743
|
}
|
|
@@ -2699,10 +2750,10 @@ class FileManager extends core_1.Host {
|
|
|
2699
2750
|
file.invalid = true;
|
|
2700
2751
|
}
|
|
2701
2752
|
else {
|
|
2702
|
-
const output =
|
|
2753
|
+
const output = renameExt(localUri, ext);
|
|
2703
2754
|
if (localUri !== output) {
|
|
2704
2755
|
try {
|
|
2705
|
-
const success =
|
|
2756
|
+
const success = Host.renameFile(localUri, output);
|
|
2706
2757
|
this.replace(file, output, mimeType);
|
|
2707
2758
|
if (!success) {
|
|
2708
2759
|
this.filesToRemove.add(localUri);
|
|
@@ -2722,43 +2773,44 @@ class FileManager extends core_1.Host {
|
|
|
2722
2773
|
async compressFile(file, overwrite = true) {
|
|
2723
2774
|
const compress = this.Compress;
|
|
2724
2775
|
const localUri = file.localUri;
|
|
2725
|
-
if (compress && localUri && (this.has(localUri) ||
|
|
2776
|
+
if (compress && localUri && (this.has(localUri) || existsFlag(file)) && file.compress) {
|
|
2726
2777
|
const tasks = [];
|
|
2727
2778
|
for (const options of file.compress) {
|
|
2728
2779
|
const { format, condition } = options;
|
|
2729
2780
|
let output;
|
|
2730
2781
|
switch (format) {
|
|
2731
2782
|
case 'br':
|
|
2783
|
+
case 'zst':
|
|
2732
2784
|
options.mimeType ||= file.mimeType;
|
|
2733
2785
|
case 'gz':
|
|
2734
2786
|
output = localUri + '.' + format;
|
|
2735
2787
|
break;
|
|
2736
2788
|
case 'woff':
|
|
2737
2789
|
case 'woff2':
|
|
2738
|
-
output =
|
|
2790
|
+
output = renameExt(localUri, format);
|
|
2739
2791
|
options.filename = file.filename;
|
|
2740
2792
|
options.etag = file.etag;
|
|
2741
2793
|
break;
|
|
2742
2794
|
default:
|
|
2743
2795
|
if (format && typeof compress.compressors[format] === 'function') {
|
|
2744
|
-
output =
|
|
2796
|
+
output = renameExt(localUri, format);
|
|
2745
2797
|
}
|
|
2746
2798
|
break;
|
|
2747
2799
|
}
|
|
2748
2800
|
if (output && (!condition || withinSizeRange(localUri, condition))) {
|
|
2749
2801
|
try {
|
|
2750
|
-
if (overwrite || !fs.existsSync(output) || !
|
|
2802
|
+
if (overwrite || !fs.existsSync(output) || !existsFlag(file) && fs.statSync(output).mtimeMs < this.startTime) {
|
|
2751
2803
|
options.timeout = this.#getModuleTimeout('compress');
|
|
2752
2804
|
tasks.push(compress.tryFile(file.buffer || localUri, output, options).then(() => {
|
|
2753
2805
|
const outFile = options.outFile;
|
|
2754
2806
|
if (outFile) {
|
|
2755
|
-
if (condition?.includes('%') &&
|
|
2807
|
+
if (condition?.includes('%') && getSize(outFile) >= getSize(localUri)) {
|
|
2756
2808
|
queueMicrotask(() => {
|
|
2757
2809
|
this.deleteFile(outFile);
|
|
2758
2810
|
});
|
|
2759
2811
|
}
|
|
2760
2812
|
else {
|
|
2761
|
-
this.add(outFile, file,
|
|
2813
|
+
this.add(outFile, file, FILE_TYPE.COMPRESSED);
|
|
2762
2814
|
}
|
|
2763
2815
|
}
|
|
2764
2816
|
})
|
|
@@ -2819,13 +2871,13 @@ class FileManager extends core_1.Host {
|
|
|
2819
2871
|
}
|
|
2820
2872
|
}
|
|
2821
2873
|
}
|
|
2822
|
-
if (
|
|
2874
|
+
if (Image.isBinary(mimeType)) {
|
|
2823
2875
|
if (file.commands && this.Image) {
|
|
2824
2876
|
const handler = this.Image.get(mimeType) || this.Image.get('handler');
|
|
2825
2877
|
if (handler) {
|
|
2826
2878
|
const { instance } = handler;
|
|
2827
2879
|
for (const command of file.commands) {
|
|
2828
|
-
if (withinSizeRange(localUri, command, REGEXP_SIZERANGE) && instance.using && !data.aborted) {
|
|
2880
|
+
if (command && withinSizeRange(localUri, command, REGEXP_SIZERANGE) && instance.using && !data.aborted) {
|
|
2829
2881
|
if (instance.threadable) {
|
|
2830
2882
|
if (this.openThread(instance, data, this.getProcessTimeout(handler))) {
|
|
2831
2883
|
instance.using(data, command)
|
|
@@ -2865,7 +2917,7 @@ class FileManager extends core_1.Host {
|
|
|
2865
2917
|
if (!file.invalid) {
|
|
2866
2918
|
this.add(localUri, parent);
|
|
2867
2919
|
}
|
|
2868
|
-
else if (localUri &&
|
|
2920
|
+
else if (localUri && isEmpty(file.bundleId)) {
|
|
2869
2921
|
this.deleteFile(localUri, { emptyDir: true });
|
|
2870
2922
|
}
|
|
2871
2923
|
});
|
|
@@ -2875,7 +2927,7 @@ class FileManager extends core_1.Host {
|
|
|
2875
2927
|
if (this.aborted || data.aborted) {
|
|
2876
2928
|
return false;
|
|
2877
2929
|
}
|
|
2878
|
-
|
|
2930
|
+
AbortComponent.attach(data, instance.signal);
|
|
2879
2931
|
if (timeout > 0) {
|
|
2880
2932
|
this.addProcessTimeout(instance, data.file, timeout);
|
|
2881
2933
|
}
|
|
@@ -2888,7 +2940,7 @@ class FileManager extends core_1.Host {
|
|
|
2888
2940
|
return false;
|
|
2889
2941
|
}
|
|
2890
2942
|
if (instance) {
|
|
2891
|
-
|
|
2943
|
+
AbortComponent.detach(data, instance.signal);
|
|
2892
2944
|
this.removeProcessTimeout(instance, data.file);
|
|
2893
2945
|
}
|
|
2894
2946
|
if (typeof callback === 'function') {
|
|
@@ -2958,7 +3010,7 @@ class FileManager extends core_1.Host {
|
|
|
2958
3010
|
result = value;
|
|
2959
3011
|
}
|
|
2960
3012
|
else if (typeof value === 'string') {
|
|
2961
|
-
if ((result =
|
|
3013
|
+
if ((result = byteLength(value, encoding)) === 0) {
|
|
2962
3014
|
return 0;
|
|
2963
3015
|
}
|
|
2964
3016
|
}
|
|
@@ -2980,20 +3032,20 @@ class FileManager extends core_1.Host {
|
|
|
2980
3032
|
}
|
|
2981
3033
|
checkHash(checksum, data, uri) {
|
|
2982
3034
|
let algorithm, digestEncoding, value;
|
|
2983
|
-
if (
|
|
2984
|
-
({ algorithm, digestEncoding
|
|
3035
|
+
if (isObject(checksum)) {
|
|
3036
|
+
({ algorithm, digestEncoding, value } = checksum);
|
|
2985
3037
|
}
|
|
2986
3038
|
else {
|
|
2987
3039
|
value = checksum;
|
|
2988
3040
|
}
|
|
2989
3041
|
let output;
|
|
2990
|
-
if (
|
|
3042
|
+
if (isPlainObject(data)) {
|
|
2991
3043
|
({ data, uri, output } = data);
|
|
2992
3044
|
}
|
|
2993
|
-
else if (
|
|
3045
|
+
else if (isPlainObject(uri)) {
|
|
2994
3046
|
({ uri, output } = uri);
|
|
2995
3047
|
}
|
|
2996
|
-
if (
|
|
3048
|
+
if (isString(value)) {
|
|
2997
3049
|
algorithm ||= "sha256";
|
|
2998
3050
|
if (!data) {
|
|
2999
3051
|
if (!uri) {
|
|
@@ -3004,14 +3056,14 @@ class FileManager extends core_1.Host {
|
|
|
3004
3056
|
}
|
|
3005
3057
|
catch (err) {
|
|
3006
3058
|
this.addLog(3, err, { source: `${algorithm}:${value}` });
|
|
3007
|
-
return output ?
|
|
3059
|
+
return output ? isErrorCode(err, 'ENOENT') : false;
|
|
3008
3060
|
}
|
|
3009
3061
|
}
|
|
3010
|
-
if ((value = value.toLowerCase()) ===
|
|
3062
|
+
if ((value = value.toLowerCase()) === Host.asHash(data, algorithm, digestEncoding)) {
|
|
3011
3063
|
if (uri instanceof URL) {
|
|
3012
|
-
uri =
|
|
3064
|
+
uri = fileURLToPath(uri);
|
|
3013
3065
|
}
|
|
3014
|
-
this.formatMessage(32, algorithm, ["Checksum matched" + (output ? ' (output)' : ''), uri && path.basename(uri)], value, { ...
|
|
3066
|
+
this.formatMessage(32, algorithm, ["Checksum matched" + (output ? ' (output)' : ''), uri && path.basename(uri)], value, { ...Host.LOG_STYLE_INFO, queue: true });
|
|
3015
3067
|
return true;
|
|
3016
3068
|
}
|
|
3017
3069
|
return false;
|
|
@@ -3039,7 +3091,7 @@ class FileManager extends core_1.Host {
|
|
|
3039
3091
|
if (buffer) {
|
|
3040
3092
|
this.addDownload(Buffer.byteLength(buffer, encoding));
|
|
3041
3093
|
}
|
|
3042
|
-
else if (
|
|
3094
|
+
else if (isString(pipeTo) && Host.isPath(pipeTo, true)) {
|
|
3043
3095
|
this.addDownload(pipeTo);
|
|
3044
3096
|
}
|
|
3045
3097
|
return buffer;
|
|
@@ -3052,7 +3104,7 @@ class FileManager extends core_1.Host {
|
|
|
3052
3104
|
else {
|
|
3053
3105
|
options.pathname ||= this.baseDirectory;
|
|
3054
3106
|
}
|
|
3055
|
-
return
|
|
3107
|
+
return Request.isRclone(uri) ? this.Request.rclone(uri, options) : this.Request.aria2c(uri, options);
|
|
3056
3108
|
}
|
|
3057
3109
|
async start(emptyDir) {
|
|
3058
3110
|
if (this.aborted) {
|
|
@@ -3179,7 +3231,7 @@ class FileManager extends core_1.Host {
|
|
|
3179
3231
|
this.deleteFile(localUri, { id: file.id, emptyDir: true, all: true });
|
|
3180
3232
|
}
|
|
3181
3233
|
}
|
|
3182
|
-
(moduleName && this.find(moduleName) || this).formatFail(16384, "FAIL!", ["Unable to process file", moduleName || 'main'],
|
|
3234
|
+
(moduleName && this.find(moduleName) || this).formatFail(16384, "FAIL!", ["Unable to process file", moduleName || 'main'], errorValue(expired ? "Timeout was exceeded" : "Aborted", localUri), { queue: false, startTime: data.startTime });
|
|
3183
3235
|
processing.splice(i--, 1);
|
|
3184
3236
|
aborted = true;
|
|
3185
3237
|
}
|
|
@@ -3190,12 +3242,12 @@ class FileManager extends core_1.Host {
|
|
|
3190
3242
|
else if (aborted) {
|
|
3191
3243
|
this.performFinalize();
|
|
3192
3244
|
}
|
|
3193
|
-
},
|
|
3245
|
+
}, THRESHOLD.FILEMANAGER_INTERVAL);
|
|
3194
3246
|
const { assets, cacheToDisk, cacheToMemory, fetchedAssets } = this;
|
|
3195
3247
|
const downloadable = new Map();
|
|
3196
3248
|
const staging = this.#incremental === "staging";
|
|
3197
3249
|
const incremental = this.config.incremental;
|
|
3198
|
-
const checkDest = (src, localUri) => staging || !
|
|
3250
|
+
const checkDest = (src, localUri) => staging || !hasSameStat(src, localUri);
|
|
3199
3251
|
let cacheable = false, cacheOpen = false, cacheEtag = false;
|
|
3200
3252
|
if (!staging) {
|
|
3201
3253
|
cacheable = this.retryLimit === RETRY_LIMIT;
|
|
@@ -3211,9 +3263,9 @@ class FileManager extends core_1.Host {
|
|
|
3211
3263
|
}
|
|
3212
3264
|
const targeting = this._usingObjects.size > 0;
|
|
3213
3265
|
const groupData = new ProcessGroup();
|
|
3214
|
-
const imported = assets.filter(item => item.imported &&
|
|
3266
|
+
const imported = assets.filter(item => item.imported && isArray(item.imported));
|
|
3215
3267
|
for (let item of assets) {
|
|
3216
|
-
if (item.invalid ||
|
|
3268
|
+
if (item.invalid || ignoreFlag(item)) {
|
|
3217
3269
|
continue;
|
|
3218
3270
|
}
|
|
3219
3271
|
let download = this.setLocalUri(item), uri = item.uri;
|
|
@@ -3224,7 +3276,7 @@ class FileManager extends core_1.Host {
|
|
|
3224
3276
|
}
|
|
3225
3277
|
item.filename = '';
|
|
3226
3278
|
}
|
|
3227
|
-
if (targeting && !this.contains(item, (other) =>
|
|
3279
|
+
if (targeting && !this.contains(item, (other) => isEqual(item, other))) {
|
|
3228
3280
|
item.invalid = true;
|
|
3229
3281
|
continue;
|
|
3230
3282
|
}
|
|
@@ -3253,7 +3305,7 @@ class FileManager extends core_1.Host {
|
|
|
3253
3305
|
if (etag && imported.some(file => file.imported.includes(uri))) {
|
|
3254
3306
|
item.incremental = "none";
|
|
3255
3307
|
}
|
|
3256
|
-
else if (!
|
|
3308
|
+
else if (!mainFlag(item)) {
|
|
3257
3309
|
let cached;
|
|
3258
3310
|
const { bundleId, bundleIndex = -1 } = item;
|
|
3259
3311
|
const setBuffer = (target) => {
|
|
@@ -3264,12 +3316,12 @@ class FileManager extends core_1.Host {
|
|
|
3264
3316
|
}
|
|
3265
3317
|
return false;
|
|
3266
3318
|
};
|
|
3267
|
-
if ((!etag || item.fetchType === 1 && (
|
|
3319
|
+
if ((!etag || item.fetchType === 1 && (isEmpty(bundleId) || bundleIndex <= 0)) && (!this.Watch || !item.watch || setBuffer(item))) {
|
|
3268
3320
|
let childBundle, childDownload;
|
|
3269
|
-
if (!
|
|
3321
|
+
if (!isEmpty(bundleId) && bundleIndex > 0) {
|
|
3270
3322
|
const target = assets.find(parent => parent.bundleIndex === 0 && parent.bundleId === bundleId);
|
|
3271
3323
|
if (target) {
|
|
3272
|
-
if (
|
|
3324
|
+
if (existsFlag(target)) {
|
|
3273
3325
|
setBuffer(item);
|
|
3274
3326
|
item.invalid = true;
|
|
3275
3327
|
}
|
|
@@ -3300,10 +3352,10 @@ class FileManager extends core_1.Host {
|
|
|
3300
3352
|
setBuffer(item);
|
|
3301
3353
|
}
|
|
3302
3354
|
const checksumOutput = item.checksumOutput;
|
|
3303
|
-
if (
|
|
3355
|
+
if (Host.isPath(localUri) && (!checksumOutput || this.checkHash(checksumOutput, { uri: localUri, output: true }))) {
|
|
3304
3356
|
item.flags |= 128;
|
|
3305
3357
|
if (!etag || item.fetchType !== 1 || checksumOutput) {
|
|
3306
|
-
if (bundleIndex === 0 && !
|
|
3358
|
+
if (bundleIndex === 0 && !isEmpty(bundleId)) {
|
|
3307
3359
|
for (const target of assets.filter(child => child.bundleId === bundleId && child.bundleIndex > 0)) {
|
|
3308
3360
|
setBuffer(target);
|
|
3309
3361
|
target.invalid = true;
|
|
@@ -3323,7 +3375,7 @@ class FileManager extends core_1.Host {
|
|
|
3323
3375
|
if (item.willChange && Buffer.isBuffer(buffer)) {
|
|
3324
3376
|
item.buffer = buffer;
|
|
3325
3377
|
}
|
|
3326
|
-
if (!(checksumOutput &&
|
|
3378
|
+
if (!(checksumOutput && isEmpty(bundleId) && this.checkHash(checksumOutput, { data: buffer, uri: localUri, output: true }))) {
|
|
3327
3379
|
this.performAsyncTask();
|
|
3328
3380
|
const target = new ProcessFile(this, item, localUri, groupData);
|
|
3329
3381
|
target.received(null, incremental, true, false, true);
|
|
@@ -3336,14 +3388,14 @@ class FileManager extends core_1.Host {
|
|
|
3336
3388
|
}
|
|
3337
3389
|
}
|
|
3338
3390
|
}
|
|
3339
|
-
else if (
|
|
3391
|
+
else if (Host.isPath(localUri)) {
|
|
3340
3392
|
try {
|
|
3341
3393
|
const buffer = fs.readFileSync(localUri);
|
|
3342
3394
|
if (!etag) {
|
|
3343
|
-
|
|
3395
|
+
setInitialValue(item, false).buffer = buffer;
|
|
3344
3396
|
}
|
|
3345
3397
|
else if (uri && CACHE_ETAG.has(uri)) {
|
|
3346
|
-
const initialValue =
|
|
3398
|
+
const initialValue = setInitialValue(item, false);
|
|
3347
3399
|
initialValue.buffer = buffer;
|
|
3348
3400
|
initialValue.etag = CACHE_ETAG.get(uri);
|
|
3349
3401
|
}
|
|
@@ -3372,13 +3424,13 @@ class FileManager extends core_1.Host {
|
|
|
3372
3424
|
}
|
|
3373
3425
|
const { pathname, localUri } = download;
|
|
3374
3426
|
const target = new ProcessFile(this, item, localUri, groupData);
|
|
3375
|
-
if (diffSource &&
|
|
3427
|
+
if (diffSource && Host.isPath(localUri, true) && diffSource[0].some(pattern => typeof pattern === 'string' ? pattern === item.mimeType || pattern === item.mimeType?.split('/')[1] : pattern(localUri))) {
|
|
3376
3428
|
try {
|
|
3377
|
-
const encoding =
|
|
3429
|
+
const encoding = Image.isBinary(item.mimeType) ? 'base64' : item.encoding || 'utf8';
|
|
3378
3430
|
diffSource[1][localUri] ||= [fs.readFileSync(localUri, encoding), encoding];
|
|
3379
3431
|
}
|
|
3380
3432
|
catch (err) {
|
|
3381
|
-
this.formatMessage(32, "WARN!", ["Unable to read file", path.basename(localUri)], err, { ...
|
|
3433
|
+
this.formatMessage(32, "WARN!", ["Unable to read file", path.basename(localUri)], err, { ...Host.LOG_STYLE_WARN });
|
|
3382
3434
|
}
|
|
3383
3435
|
}
|
|
3384
3436
|
if (item.content) {
|
|
@@ -3425,7 +3477,7 @@ class FileManager extends core_1.Host {
|
|
|
3425
3477
|
target.received(err);
|
|
3426
3478
|
}).then(code => {
|
|
3427
3479
|
if (code === -1) {
|
|
3428
|
-
target.received(
|
|
3480
|
+
target.received(errorValue("Not able to read buffer", path.basename(localUri)));
|
|
3429
3481
|
}
|
|
3430
3482
|
});
|
|
3431
3483
|
}
|
|
@@ -3435,10 +3487,10 @@ class FileManager extends core_1.Host {
|
|
|
3435
3487
|
continue;
|
|
3436
3488
|
}
|
|
3437
3489
|
const type = item.fetchType || 0;
|
|
3438
|
-
const bundleMain = item.bundleIndex === 0 && !
|
|
3490
|
+
const bundleMain = item.bundleIndex === 0 && !isEmpty(item.bundleId);
|
|
3439
3491
|
if (type === 1 || type === 2) {
|
|
3440
|
-
let checkEtag;
|
|
3441
|
-
if (bundling[uri] &&
|
|
3492
|
+
let checkEtag = false;
|
|
3493
|
+
if (bundling[uri] && isEmpty(item.bundleId)) {
|
|
3442
3494
|
bundling[uri].push(item);
|
|
3443
3495
|
}
|
|
3444
3496
|
else if ((checkEtag = hasEtag(item, cacheable, cacheEtag)) || !target.queued(pathname, emptyDir, false)) {
|
|
@@ -3460,7 +3512,7 @@ class FileManager extends core_1.Host {
|
|
|
3460
3512
|
if (type === 1) {
|
|
3461
3513
|
cacheDir = cacheToDisk.has(url) && isCacheable(item);
|
|
3462
3514
|
cacheBuffer = cacheToMemory.has(url);
|
|
3463
|
-
mainEtag =
|
|
3515
|
+
mainEtag = mainFlag(item) && (cacheEtag || item.incremental === "etag");
|
|
3464
3516
|
}
|
|
3465
3517
|
const downloadUri = (request, etagDir) => {
|
|
3466
3518
|
if (checkEtag) {
|
|
@@ -3468,14 +3520,17 @@ class FileManager extends core_1.Host {
|
|
|
3468
3520
|
}
|
|
3469
3521
|
client?.destroy();
|
|
3470
3522
|
const location = request.url.toString();
|
|
3523
|
+
let maxAge;
|
|
3471
3524
|
request.encoding = encoding;
|
|
3525
|
+
request.headers = item.headers;
|
|
3472
3526
|
request.pipeTo = localUri;
|
|
3473
3527
|
request.statusMessage = location + (bundleMain ? ' (0)' : '');
|
|
3474
3528
|
if (type === 1) {
|
|
3475
3529
|
request.method = 'GET';
|
|
3476
3530
|
request.httpVersion = (originCount[request.url.origin] || 0) <= 1 ? 1 : undefined;
|
|
3477
3531
|
request.connected = headers => {
|
|
3478
|
-
processHeaders(
|
|
3532
|
+
processHeaders(headers, item, watching, mainEtag);
|
|
3533
|
+
maxAge = processCacheHeaders(headers, request.noCache, request.noStore);
|
|
3479
3534
|
return item.willChange || !!etagDir && (cacheDir || !item.contentLength || cacheToMemory.within(item.contentLength)) || this.hasLog('progress');
|
|
3480
3535
|
};
|
|
3481
3536
|
}
|
|
@@ -3492,13 +3547,13 @@ class FileManager extends core_1.Host {
|
|
|
3492
3547
|
item.buffer = data;
|
|
3493
3548
|
}
|
|
3494
3549
|
target.received(null, incremental, type === 1);
|
|
3495
|
-
if (etagDir) {
|
|
3550
|
+
if (etagDir && isStored(request.noCache, maxAge)) {
|
|
3496
3551
|
queueMicrotask(() => {
|
|
3497
3552
|
if (cacheBuffer && data) {
|
|
3498
|
-
cacheToMemory.add(location, etagDir, data, {
|
|
3553
|
+
cacheToMemory.add(location, etagDir, data, { contentLength: item.contentLength, maxAge, encoding, toDisk: !cacheDir && isCacheable(item) ? path.basename(localUri) : '' });
|
|
3499
3554
|
}
|
|
3500
3555
|
if (cacheDir) {
|
|
3501
|
-
cacheToDisk.add(location, etagDir, localUri, {
|
|
3556
|
+
cacheToDisk.add(location, etagDir, localUri, { contentLength: item.contentLength, maxAge, buffer: data });
|
|
3502
3557
|
}
|
|
3503
3558
|
});
|
|
3504
3559
|
}
|
|
@@ -3512,15 +3567,15 @@ class FileManager extends core_1.Host {
|
|
|
3512
3567
|
const request = this.Request;
|
|
3513
3568
|
try {
|
|
3514
3569
|
++processTask.pending;
|
|
3515
|
-
const opts = request.opts(href, { method: 'HEAD', httpVersion: 1 });
|
|
3570
|
+
const opts = request.opts(href, { method: 'HEAD', httpVersion: 1, headers: item.headers });
|
|
3516
3571
|
(client = request.open(href, opts))
|
|
3517
3572
|
.on('response', res => {
|
|
3518
3573
|
--processTask.pending;
|
|
3519
3574
|
const statusCode = res.statusCode;
|
|
3520
3575
|
if (statusCode < 300) {
|
|
3521
|
-
const etag = processHeaders(
|
|
3576
|
+
const etag = processHeaders(res.headers, item, false, mainEtag);
|
|
3522
3577
|
let etagDir = null, tempDir;
|
|
3523
|
-
if (etag) {
|
|
3578
|
+
if (etag && !opts.noCache) {
|
|
3524
3579
|
if (cacheDir || mainEtag) {
|
|
3525
3580
|
tempDir = this.getCacheDir(href);
|
|
3526
3581
|
}
|
|
@@ -3552,7 +3607,7 @@ class FileManager extends core_1.Host {
|
|
|
3552
3607
|
}
|
|
3553
3608
|
try {
|
|
3554
3609
|
let pipeAs;
|
|
3555
|
-
if (tempDir &&
|
|
3610
|
+
if (tempDir && hasSize(pipeAs = path.join(tempDir, etagDir, path.basename(localUri)))) {
|
|
3556
3611
|
if (!checkEtag) {
|
|
3557
3612
|
if (cacheBuffer && !buffer) {
|
|
3558
3613
|
buffer = fs.readFileSync(pipeAs, encoding);
|
|
@@ -3561,7 +3616,7 @@ class FileManager extends core_1.Host {
|
|
|
3561
3616
|
return;
|
|
3562
3617
|
}
|
|
3563
3618
|
if (cacheToMemory.within(item.contentLength = Buffer.byteLength(buffer, encoding))) {
|
|
3564
|
-
cacheToMemory.add(location, etagDir, buffer, {
|
|
3619
|
+
cacheToMemory.add(location, etagDir, buffer, { contentLength: item.contentLength, encoding, tempFile: pipeAs });
|
|
3565
3620
|
}
|
|
3566
3621
|
}
|
|
3567
3622
|
if (checkDest(pipeAs, localUri)) {
|
|
@@ -3575,12 +3630,12 @@ class FileManager extends core_1.Host {
|
|
|
3575
3630
|
}
|
|
3576
3631
|
else {
|
|
3577
3632
|
fs.copyFileSync(pipeAs, localUri);
|
|
3578
|
-
this.addDownload(pipeAs,
|
|
3633
|
+
this.addDownload(pipeAs, DOWNLOAD_TYPE.CACHE);
|
|
3579
3634
|
}
|
|
3580
3635
|
}
|
|
3581
3636
|
if (buffer) {
|
|
3582
3637
|
fs.writeFileSync(localUri, buffer);
|
|
3583
|
-
this.addDownload(Buffer.byteLength(buffer, encoding),
|
|
3638
|
+
this.addDownload(Buffer.byteLength(buffer, encoding), DOWNLOAD_TYPE.CACHE);
|
|
3584
3639
|
}
|
|
3585
3640
|
}
|
|
3586
3641
|
if (item.willChange && buffer) {
|
|
@@ -3597,7 +3652,7 @@ class FileManager extends core_1.Host {
|
|
|
3597
3652
|
if (buffer) {
|
|
3598
3653
|
setContent(item, buffer);
|
|
3599
3654
|
fs.writeFileSync(localUri, buffer);
|
|
3600
|
-
this.addDownload(Buffer.byteLength(buffer, encoding),
|
|
3655
|
+
this.addDownload(Buffer.byteLength(buffer, encoding), DOWNLOAD_TYPE.CACHE);
|
|
3601
3656
|
if (checkEtag) {
|
|
3602
3657
|
item.flags &= ~128;
|
|
3603
3658
|
}
|
|
@@ -3620,13 +3675,13 @@ class FileManager extends core_1.Host {
|
|
|
3620
3675
|
const location = res.headers.location;
|
|
3621
3676
|
if (location && ++redirects <= HTTP_CLIENT.redirectLimit) {
|
|
3622
3677
|
try {
|
|
3623
|
-
checkHeaders(new URL(
|
|
3678
|
+
checkHeaders(new URL(fromURL(href, location)), redirects);
|
|
3624
3679
|
return;
|
|
3625
3680
|
}
|
|
3626
3681
|
catch {
|
|
3627
3682
|
}
|
|
3628
3683
|
}
|
|
3629
|
-
target.abort(
|
|
3684
|
+
target.abort(errorMessage(statusCode, "Exceeded redirect limit", url.toString()));
|
|
3630
3685
|
}
|
|
3631
3686
|
else {
|
|
3632
3687
|
downloadUri(opts, null);
|
|
@@ -3635,7 +3690,7 @@ class FileManager extends core_1.Host {
|
|
|
3635
3690
|
.on('error', err => {
|
|
3636
3691
|
--processTask.pending;
|
|
3637
3692
|
if (!client.destroyed) {
|
|
3638
|
-
if (
|
|
3693
|
+
if (checkRetryable(err)) {
|
|
3639
3694
|
downloadUri(opts, null);
|
|
3640
3695
|
}
|
|
3641
3696
|
else {
|
|
@@ -3665,7 +3720,7 @@ class FileManager extends core_1.Host {
|
|
|
3665
3720
|
}
|
|
3666
3721
|
else if (hasFiles(type, uri)) {
|
|
3667
3722
|
if (bundleMain && !item.mimeType) {
|
|
3668
|
-
target.abort(
|
|
3723
|
+
target.abort(errorValue("MIME not found", uri), true);
|
|
3669
3724
|
}
|
|
3670
3725
|
else if (!target.queued(pathname, emptyDir, false)) {
|
|
3671
3726
|
if (downloading[uri]) {
|
|
@@ -3674,7 +3729,7 @@ class FileManager extends core_1.Host {
|
|
|
3674
3729
|
}
|
|
3675
3730
|
downloading[uri] = [];
|
|
3676
3731
|
this.performAsyncTask();
|
|
3677
|
-
const options = { pathname, binOpts: item.binOpts, shellExpansion: item.shellExpansion };
|
|
3732
|
+
const options = { pathname, binOpts: item.binOpts, shellExpansion: item.shellExpansion, headers: item.headers };
|
|
3678
3733
|
let src;
|
|
3679
3734
|
if (type === 0) {
|
|
3680
3735
|
if (item.filename) {
|
|
@@ -3688,7 +3743,7 @@ class FileManager extends core_1.Host {
|
|
|
3688
3743
|
}
|
|
3689
3744
|
void this.scheduleTask(src, options, (result) => {
|
|
3690
3745
|
if (result.length === 0) {
|
|
3691
|
-
target.abort(
|
|
3746
|
+
target.abort(errorValue("No files were successfully downloaded", uri));
|
|
3692
3747
|
}
|
|
3693
3748
|
else if (bundleMain) {
|
|
3694
3749
|
const encoding = item.encoding ||= 'utf8';
|
|
@@ -3719,7 +3774,7 @@ class FileManager extends core_1.Host {
|
|
|
3719
3774
|
const file = result[i];
|
|
3720
3775
|
if (index === i || !found && ext === path.extname(file)) {
|
|
3721
3776
|
try {
|
|
3722
|
-
if (file !== localUri && !
|
|
3777
|
+
if (file !== localUri && !Host.renameFile(file, localUri)) {
|
|
3723
3778
|
this.filesToRemove.add(file);
|
|
3724
3779
|
}
|
|
3725
3780
|
found = true;
|
|
@@ -3729,7 +3784,7 @@ class FileManager extends core_1.Host {
|
|
|
3729
3784
|
}
|
|
3730
3785
|
}
|
|
3731
3786
|
else {
|
|
3732
|
-
this.add(file, item,
|
|
3787
|
+
this.add(file, item, FILE_TYPE.TORRENT);
|
|
3733
3788
|
}
|
|
3734
3789
|
}
|
|
3735
3790
|
}
|
|
@@ -3759,7 +3814,7 @@ class FileManager extends core_1.Host {
|
|
|
3759
3814
|
if (checkDest(uri, localUri)) {
|
|
3760
3815
|
fs.copyFile(uri, localUri, err => {
|
|
3761
3816
|
if (!err) {
|
|
3762
|
-
this.addDownload(localUri,
|
|
3817
|
+
this.addDownload(localUri, DOWNLOAD_TYPE.DISK);
|
|
3763
3818
|
}
|
|
3764
3819
|
target.received(err);
|
|
3765
3820
|
});
|
|
@@ -3802,12 +3857,12 @@ class FileManager extends core_1.Host {
|
|
|
3802
3857
|
else if (processTask.updated + 100 > currentTime) {
|
|
3803
3858
|
return;
|
|
3804
3859
|
}
|
|
3805
|
-
const logCurrent =
|
|
3860
|
+
const logCurrent = getLogCurrent();
|
|
3806
3861
|
const statusHeight = status.length;
|
|
3807
3862
|
const barLength = processTask.barLength;
|
|
3808
3863
|
const barIncrement = 100 / barLength;
|
|
3809
3864
|
const initial = processTask.logCurrent !== logCurrent;
|
|
3810
|
-
let redraw = false, scrollHeight = 0, logDelayed =
|
|
3865
|
+
let redraw = false, scrollHeight = 0, logDelayed = Host.getLogDelayed();
|
|
3811
3866
|
if (logDelayed.length > 0) {
|
|
3812
3867
|
logDelayed = logDelayed.filter(item => typeof item[4].progressBar !== 'boolean');
|
|
3813
3868
|
scrollHeight = Math.min(logDelayed.length, LOGGER.PROGRESS_SCROLLBUFFER);
|
|
@@ -3887,7 +3942,7 @@ class FileManager extends core_1.Host {
|
|
|
3887
3942
|
}
|
|
3888
3943
|
const diffTime = currentTime - item.startTime;
|
|
3889
3944
|
const unit = receivedBytes < 1048576 ? 'KB' : receivedBytes < 1073741824 ? 'MB' : 'GB';
|
|
3890
|
-
const [time, size, speed] = formatSegment(endTime > 0 ? bars === 0 ? -1 : endTime : 0, formatMinutes(diffTime).padStart(LOGGER.TITLE_WIDTH),
|
|
3945
|
+
const [time, size, speed] = formatSegment(endTime > 0 ? bars === 0 ? -1 : endTime : 0, formatMinutes(diffTime).padStart(LOGGER.TITLE_WIDTH), formatSize(receivedBytes, { unit, unitSeparator: ' ', fixedDecimals: true, decimalPlaces: unit === 'KB' ? 0 : unit === 'MB' ? 1 : 2 }).padStart(8), getTransferRate(receivedBytes, item.dataTime ? Math.max(1, convertTime(process.hrtime.bigint() - item.dataTime, false) * 1000) : diffTime, ' ').padStart(10));
|
|
3891
3946
|
let output = time + ` ${chalk.blackBright(LOGGER.TITLE_SEP)} ` + (endTime > 0 && bars === 0 ? chalk.grey(item.url) : item.url) + spacer;
|
|
3892
3947
|
if (barLength === 1) {
|
|
3893
3948
|
output += ` ${endTime > 0 ? bars === 0 ? chalk.red('ERR!') : LOGGER.PROGRESS_COMPLETE : formatPercent(percent)} ` + spacer;
|
|
@@ -3906,7 +3961,7 @@ class FileManager extends core_1.Host {
|
|
|
3906
3961
|
}
|
|
3907
3962
|
output += '\n';
|
|
3908
3963
|
if (!LOGGER.PROGRESS_USECOLOR) {
|
|
3909
|
-
output =
|
|
3964
|
+
output = stripVTControlCharacters(output);
|
|
3910
3965
|
}
|
|
3911
3966
|
PROCESS_STDOUT.clearLine(0);
|
|
3912
3967
|
PROCESS_STDOUT.write(output);
|
|
@@ -3964,7 +4019,7 @@ class FileManager extends core_1.Host {
|
|
|
3964
4019
|
item[4] = options;
|
|
3965
4020
|
}
|
|
3966
4021
|
PROCESS_STDOUT.clearLine(0);
|
|
3967
|
-
|
|
4022
|
+
Host.formatMessage(...item);
|
|
3968
4023
|
output = true;
|
|
3969
4024
|
}
|
|
3970
4025
|
if (last) {
|
|
@@ -3981,7 +4036,7 @@ class FileManager extends core_1.Host {
|
|
|
3981
4036
|
}
|
|
3982
4037
|
}
|
|
3983
4038
|
else if (output) {
|
|
3984
|
-
processTask.logCurrent =
|
|
4039
|
+
processTask.logCurrent = getLogCurrent();
|
|
3985
4040
|
}
|
|
3986
4041
|
}
|
|
3987
4042
|
if (cleared) {
|
|
@@ -4010,7 +4065,7 @@ class FileManager extends core_1.Host {
|
|
|
4010
4065
|
return Promise.resolve(0);
|
|
4011
4066
|
}
|
|
4012
4067
|
let id = 0, target;
|
|
4013
|
-
if (
|
|
4068
|
+
if (isPlainObject(data)) {
|
|
4014
4069
|
++scheduler.count;
|
|
4015
4070
|
if ('pathname' in data || 'binOpts' in data) {
|
|
4016
4071
|
target = this.fetchFiles(uri, data);
|
|
@@ -4029,9 +4084,9 @@ class FileManager extends core_1.Host {
|
|
|
4029
4084
|
}
|
|
4030
4085
|
else {
|
|
4031
4086
|
if (data instanceof URL && data.protocol === 'file:') {
|
|
4032
|
-
data =
|
|
4087
|
+
data = fileURLToPath(data);
|
|
4033
4088
|
}
|
|
4034
|
-
if (
|
|
4089
|
+
if (isString(data) && Host.isPath(data, true) && this.canRead(data)) {
|
|
4035
4090
|
data = fs.createReadStream(data);
|
|
4036
4091
|
}
|
|
4037
4092
|
}
|
|
@@ -4082,12 +4137,12 @@ class FileManager extends core_1.Host {
|
|
|
4082
4137
|
}
|
|
4083
4138
|
const files = [[item.localUri, mimeType]];
|
|
4084
4139
|
if (item.transforms) {
|
|
4085
|
-
files.push(...item.transforms.map(img => [img,
|
|
4140
|
+
files.push(...item.transforms.map(img => [img, Host.lookupMime(img)]));
|
|
4086
4141
|
}
|
|
4087
4142
|
tasks.push(new Promise(async (resolve) => {
|
|
4088
4143
|
for (const [file, type] of files) {
|
|
4089
4144
|
const ext = type.split('/')[1].split('+')[0];
|
|
4090
|
-
const output = item.compress.filter(({ format, condition }) => (!format && type !==
|
|
4145
|
+
const output = item.compress.filter(({ format, condition }) => (!format && type !== Image.MIME_SVG || format === ext) && (!condition || withinSizeRange(file, condition)));
|
|
4091
4146
|
if (output.length > 0 && this.has(file)) {
|
|
4092
4147
|
for (let i = 0, length = output.length, buffer = null; i < length; ++i) {
|
|
4093
4148
|
const options = output[i];
|
|
@@ -4097,7 +4152,7 @@ class FileManager extends core_1.Host {
|
|
|
4097
4152
|
.then(result => {
|
|
4098
4153
|
if (result) {
|
|
4099
4154
|
if (file === item.localUri) {
|
|
4100
|
-
item.buffer =
|
|
4155
|
+
item.buffer = Compress.asBuffer(result);
|
|
4101
4156
|
}
|
|
4102
4157
|
if (i < length - 1) {
|
|
4103
4158
|
buffer = Buffer.from(result);
|
|
@@ -4143,7 +4198,7 @@ class FileManager extends core_1.Host {
|
|
|
4143
4198
|
async finalizeCloud() {
|
|
4144
4199
|
const instance = this.Cloud;
|
|
4145
4200
|
if (instance) {
|
|
4146
|
-
await
|
|
4201
|
+
await Cloud.finalize.call(this, instance).catch((err) => this.rejectModule(err, 64, { instance }));
|
|
4147
4202
|
}
|
|
4148
4203
|
}
|
|
4149
4204
|
async finalizeChecksum() {
|
|
@@ -4154,18 +4209,18 @@ class FileManager extends core_1.Host {
|
|
|
4154
4209
|
if (typeof checksum === 'boolean' || checksum === 1) {
|
|
4155
4210
|
checksum = { recursive: checksum };
|
|
4156
4211
|
}
|
|
4157
|
-
else if (
|
|
4212
|
+
else if (isString(checksum)) {
|
|
4158
4213
|
const items = checksum.split('.');
|
|
4159
4214
|
checksum = items.length > 1 ? { algorithm: items.at(-1), filename: checksum } : { algorithm: checksum };
|
|
4160
4215
|
}
|
|
4161
|
-
if (
|
|
4216
|
+
if (isPlainObject(checksum)) {
|
|
4162
4217
|
const baseDirectory = this.baseDirectory;
|
|
4163
4218
|
checksum.joinRoot = true;
|
|
4164
4219
|
checksum.throwsEmpty = true;
|
|
4165
4220
|
const sumTime = LOG_TIMEELAPSED ? process.hrtime.bigint() : 0;
|
|
4166
4221
|
const files = (await FileManager.writeChecksum(baseDirectory, checksum.filename, checksum));
|
|
4167
4222
|
if (sumTime) {
|
|
4168
|
-
this.writeTimeElapsed(checksum.algorithm || "sha256", [baseDirectory, files.length + (files.length === 1 ? ' file' : ' files')], sumTime, { ...
|
|
4223
|
+
this.writeTimeElapsed(checksum.algorithm || "sha256", [baseDirectory, files.length + (files.length === 1 ? ' file' : ' files')], sumTime, { ...Host.LOG_STYLE_WARN });
|
|
4169
4224
|
}
|
|
4170
4225
|
}
|
|
4171
4226
|
}
|
|
@@ -4187,17 +4242,17 @@ class FileManager extends core_1.Host {
|
|
|
4187
4242
|
}
|
|
4188
4243
|
async finalize() {
|
|
4189
4244
|
if (this.aborted) {
|
|
4190
|
-
return
|
|
4245
|
+
return createAbortError(true);
|
|
4191
4246
|
}
|
|
4192
4247
|
if (LOG_TIMEELAPSED) {
|
|
4193
|
-
this.writeTimeElapsed('READY', 'Finalizing assets...', this.startTime, { ...
|
|
4248
|
+
this.writeTimeElapsed('READY', 'Finalizing assets...', this.startTime, { ...Host.LOG_STYLE_WARN, showCpu: true });
|
|
4194
4249
|
}
|
|
4195
4250
|
const startTime = process.hrtime.bigint();
|
|
4196
4251
|
for (const [file, output] of this.filesToCompare) {
|
|
4197
4252
|
const localUri = file.localUri;
|
|
4198
|
-
let minFile = localUri, minSize =
|
|
4253
|
+
let minFile = localUri, minSize = getSize(minFile);
|
|
4199
4254
|
for (const other of output) {
|
|
4200
|
-
const size =
|
|
4255
|
+
const size = getSize(other);
|
|
4201
4256
|
if (minSize === 0 || size > 0 && size < minSize) {
|
|
4202
4257
|
this.filesToRemove.add(minFile);
|
|
4203
4258
|
minFile = other;
|
|
@@ -4214,11 +4269,11 @@ class FileManager extends core_1.Host {
|
|
|
4214
4269
|
this.removeFiles();
|
|
4215
4270
|
await this.finalizeCompress(this.assets.filter(item => item.compress && !ignoreAsset(item))).catch((err) => this.rejectModule(err, 8));
|
|
4216
4271
|
if (this.aborted) {
|
|
4217
|
-
return
|
|
4272
|
+
return createAbortError(true);
|
|
4218
4273
|
}
|
|
4219
4274
|
await this.finalizeDocument().catch((err) => this.rejectModule(err, 4));
|
|
4220
4275
|
if (this.aborted) {
|
|
4221
|
-
return
|
|
4276
|
+
return createAbortError(true);
|
|
4222
4277
|
}
|
|
4223
4278
|
for (const item of this.assets) {
|
|
4224
4279
|
if (item.sourceUTF8 && !ignoreAsset(item)) {
|
|
@@ -4233,7 +4288,7 @@ class FileManager extends core_1.Host {
|
|
|
4233
4288
|
this.removeFiles();
|
|
4234
4289
|
await this.finalizeTask(this.taskAssets.filter(item => item.tasks?.find(data => !data.preceding) && this.has(item.localUri) && !ignoreAsset(item))).catch((err) => this.rejectModule(err, 4));
|
|
4235
4290
|
if (this.aborted) {
|
|
4236
|
-
return
|
|
4291
|
+
return createAbortError(true);
|
|
4237
4292
|
}
|
|
4238
4293
|
for (const item of this.assets) {
|
|
4239
4294
|
if (item.checksumOutput && !ignoreAsset(item)) {
|
|
@@ -4241,14 +4296,14 @@ class FileManager extends core_1.Host {
|
|
|
4241
4296
|
if (!this.filesToRemove.has(localUri) && !this.checkHash(item.checksumOutput, { uri: localUri, output: true })) {
|
|
4242
4297
|
item.invalid = true;
|
|
4243
4298
|
this.filesToRemove.add(localUri);
|
|
4244
|
-
this.writeFail(["Checksum did not match", path.basename(localUri)],
|
|
4299
|
+
this.writeFail(["Checksum did not match", path.basename(localUri)], errorValue(localUri, "Invalid checksum"), { type: 32, startTime, queue: true });
|
|
4245
4300
|
}
|
|
4246
4301
|
}
|
|
4247
4302
|
}
|
|
4248
4303
|
this.removeFiles();
|
|
4249
4304
|
await this.finalizeCloud().catch((err) => this.rejectModule(err, 64));
|
|
4250
4305
|
if (this.aborted) {
|
|
4251
|
-
return
|
|
4306
|
+
return createAbortError(true);
|
|
4252
4307
|
}
|
|
4253
4308
|
this.removeFiles();
|
|
4254
4309
|
if (this.Compress) {
|
|
@@ -4293,12 +4348,12 @@ class FileManager extends core_1.Host {
|
|
|
4293
4348
|
});
|
|
4294
4349
|
instance.on('dir:remove', value => {
|
|
4295
4350
|
if (this.removeCwd(value)) {
|
|
4296
|
-
value =
|
|
4351
|
+
value = toPathCase(value);
|
|
4297
4352
|
if (!value.endsWith(path.sep)) {
|
|
4298
4353
|
value += path.sep;
|
|
4299
4354
|
}
|
|
4300
4355
|
for (const name of this.files) {
|
|
4301
|
-
if (
|
|
4356
|
+
if (toPathCase(name).startsWith(value)) {
|
|
4302
4357
|
this.files.delete(name);
|
|
4303
4358
|
}
|
|
4304
4359
|
}
|
|
@@ -4307,7 +4362,7 @@ class FileManager extends core_1.Host {
|
|
|
4307
4362
|
}
|
|
4308
4363
|
startMessage() {
|
|
4309
4364
|
if (!this.silent) {
|
|
4310
|
-
this.formatMessage(128, 'START', [new Date().toLocaleString(), formatLength('asset', this.assets.length)], this.baseDirectory, { ...
|
|
4365
|
+
this.formatMessage(128, 'START', [new Date().toLocaleString(), formatLength('asset', this.assets.length)], this.baseDirectory, { ...Host.LOG_STYLE_SUCCESS });
|
|
4311
4366
|
}
|
|
4312
4367
|
}
|
|
4313
4368
|
closeMessage(startTime) {
|
|
@@ -4319,13 +4374,13 @@ class FileManager extends core_1.Host {
|
|
|
4319
4374
|
}
|
|
4320
4375
|
this.#downloadStats.forEach(([size, count], index) => {
|
|
4321
4376
|
if (count > 0) {
|
|
4322
|
-
message.push(`${
|
|
4377
|
+
message.push(`${DOWNLOAD_TYPE[index]} ` + (count > 1 && MEMORY.FILE_COUNT ? count + ' / ' : '') + formatSize(size));
|
|
4323
4378
|
}
|
|
4324
4379
|
});
|
|
4325
4380
|
if (MEMORY.SIZE > 0) {
|
|
4326
|
-
message.push('BUFFER ' + (MEMORY.CACHE.size > 1 && MEMORY.FILE_COUNT ? MEMORY.CACHE.size + ' / ' : '') +
|
|
4381
|
+
message.push('BUFFER ' + (MEMORY.CACHE.size > 1 && MEMORY.FILE_COUNT ? MEMORY.CACHE.size + ' / ' : '') + formatSize(MEMORY.SIZE));
|
|
4327
4382
|
}
|
|
4328
|
-
this.writeTimeElapsed('CLOSE', ['No further modifications', message.length > 0 ? message.join(chalk.blackBright(LOGGER.MESSAGE_SEP)) : ''], startTime, { ...
|
|
4383
|
+
this.writeTimeElapsed('CLOSE', ['No further modifications', message.length > 0 ? message.join(chalk.blackBright(LOGGER.MESSAGE_SEP)) : ''], startTime, { ...Host.LOG_STYLE_WARN });
|
|
4329
4384
|
}
|
|
4330
4385
|
}
|
|
4331
4386
|
diffMessage() {
|
|
@@ -4339,9 +4394,9 @@ class FileManager extends core_1.Host {
|
|
|
4339
4394
|
};
|
|
4340
4395
|
let divider = false;
|
|
4341
4396
|
for (const pathname in diffSource) {
|
|
4342
|
-
if (
|
|
4397
|
+
if (Host.isPath(pathname, true)) {
|
|
4343
4398
|
const [content, encoding] = diffSource[pathname];
|
|
4344
|
-
const hunks =
|
|
4399
|
+
const hunks = structuredPatch('', '', checkEOF(content), checkEOF(fs.readFileSync(pathname, encoding)), '', '', { context: 0 }).hunks;
|
|
4345
4400
|
const lastHunk = hunks.at(-1);
|
|
4346
4401
|
if (lastHunk) {
|
|
4347
4402
|
const oldMax = lastHunk.oldStart + lastHunk.oldLines;
|
|
@@ -4350,7 +4405,7 @@ class FileManager extends core_1.Host {
|
|
|
4350
4405
|
addHeader(pathname, formatLength('hunk', hunks.length));
|
|
4351
4406
|
for (const { lines, oldLines, oldStart, newStart, newLines } of hunks) {
|
|
4352
4407
|
for (let i = 0, j = oldStart, k = oldLines, l = 0, q = lines.length, added = false; i < q && k > 0; ++i) {
|
|
4353
|
-
const line = lines[i].
|
|
4408
|
+
const line = lines[i].slice(1);
|
|
4354
4409
|
output.push(` ${chalk[!line && k === 1 ? 'grey' : added ? 'green' : 'red']((j + l++).toString().padStart(posLength))} ${chalk[added ? 'green' : 'red'](line)}`);
|
|
4355
4410
|
if (--k === 0 && !added) {
|
|
4356
4411
|
j = newStart;
|
|
@@ -4373,18 +4428,18 @@ class FileManager extends core_1.Host {
|
|
|
4373
4428
|
output.push(chalk.blackBright('-'.repeat(LOGGER.MESSAGE_WIDTH)));
|
|
4374
4429
|
}
|
|
4375
4430
|
output.push('');
|
|
4376
|
-
if (this.broadcastId &&
|
|
4377
|
-
|
|
4431
|
+
if (this.broadcastId && Host.hasLogType(2)) {
|
|
4432
|
+
Host.formatMessage(2, '', '', output.join('\n'), { broadcastId: this.broadcastId, rawOutput: true });
|
|
4378
4433
|
}
|
|
4379
4434
|
else {
|
|
4380
4435
|
process.stdout.write(output.join('\n'));
|
|
4381
|
-
|
|
4436
|
+
setLogCurrent(null);
|
|
4382
4437
|
}
|
|
4383
4438
|
}
|
|
4384
4439
|
}
|
|
4385
4440
|
endMessage(errors) {
|
|
4386
4441
|
if (LOG_TIMEELAPSED) {
|
|
4387
|
-
this.writeTimeElapsed('END', this.baseDirectory, this.startTime, errors.length > 0 ? { failed: true, ...
|
|
4442
|
+
this.writeTimeElapsed('END', this.baseDirectory, this.startTime, errors.length > 0 ? { failed: true, ...Host.LOG_STYLE_FAIL } : { ...Host.LOG_STYLE_SUCCESS });
|
|
4388
4443
|
}
|
|
4389
4444
|
}
|
|
4390
4445
|
clearStorage() {
|
|
@@ -4398,7 +4453,7 @@ class FileManager extends core_1.Host {
|
|
|
4398
4453
|
const errors = instance.errors;
|
|
4399
4454
|
if (errors.length > 0) {
|
|
4400
4455
|
for (const value of errors) {
|
|
4401
|
-
outLog.push(`[${instance.moduleName}] ` + (
|
|
4456
|
+
outLog.push(`[${instance.moduleName}] ` + (Host.asString(value) || "Unknown"));
|
|
4402
4457
|
}
|
|
4403
4458
|
errors.length = 0;
|
|
4404
4459
|
}
|
|
@@ -4430,16 +4485,16 @@ class FileManager extends core_1.Host {
|
|
|
4430
4485
|
}
|
|
4431
4486
|
}
|
|
4432
4487
|
#tryPackage(target, name, type) {
|
|
4433
|
-
if (
|
|
4488
|
+
if (isString(target)) {
|
|
4434
4489
|
try {
|
|
4435
|
-
target =
|
|
4490
|
+
target = requireESM(target);
|
|
4436
4491
|
}
|
|
4437
4492
|
catch (err) {
|
|
4438
4493
|
this.checkPackage(err, target, ["Unable to load handler", this.moduleName], type);
|
|
4439
4494
|
return null;
|
|
4440
4495
|
}
|
|
4441
4496
|
}
|
|
4442
|
-
return
|
|
4497
|
+
return Host.constructorOf(target, name) ? target : null;
|
|
4443
4498
|
}
|
|
4444
4499
|
#collectErrors() {
|
|
4445
4500
|
const result = [];
|
|
@@ -4576,4 +4631,5 @@ class FileManager extends core_1.Host {
|
|
|
4576
4631
|
}
|
|
4577
4632
|
}
|
|
4578
4633
|
setLogMinWidth();
|
|
4634
|
+
|
|
4579
4635
|
module.exports = FileManager;
|