meodp 0.0.6 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bin/index.mjs +0 -0
- package/dist/cli/index.cjs +188 -18
- package/dist/cli/index.mjs +174 -7
- package/dist/index.cjs +39 -37
- package/dist/index.d.cts +87 -3
- package/dist/index.d.mts +87 -3
- package/dist/index.d.ts +87 -3
- package/dist/index.mjs +9 -8
- package/dist/shared/{meodp.c2959ba5.cjs → meodp.6b19529e.cjs} +286 -191
- package/dist/shared/{meodp.9c5df07b.mjs → meodp.b58178b8.mjs} +281 -187
- package/package.json +6 -2
|
@@ -1,37 +1,73 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
const consola = require('consola');
|
|
4
|
-
const PQueue = require('p-queue');
|
|
5
3
|
const utils = require('consola/utils');
|
|
6
4
|
const playwright = require('playwright');
|
|
7
5
|
const path = require('node:path');
|
|
8
6
|
const process = require('node:process');
|
|
9
|
-
const
|
|
10
|
-
const
|
|
11
|
-
const
|
|
12
|
-
const winston = require('winston');
|
|
7
|
+
const consola = require('consola');
|
|
8
|
+
const node = require('lowdb/node');
|
|
9
|
+
const PQueue = require('p-queue');
|
|
13
10
|
const cliProgress = require('cli-progress');
|
|
14
11
|
const Options = require('cli-progress/lib/options.js');
|
|
15
12
|
const _defaultFormatBar = require('cli-progress/lib/format-bar.js');
|
|
16
13
|
const _defaultFormatTime = require('cli-progress/lib/format-time.js');
|
|
14
|
+
const dateFns = require('date-fns');
|
|
15
|
+
const fs = require('fs-extra');
|
|
16
|
+
const stripAnsi = require('strip-ansi');
|
|
17
|
+
const winston = require('winston');
|
|
17
18
|
|
|
18
19
|
function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e.default : e; }
|
|
19
20
|
|
|
20
|
-
const consola__default = /*#__PURE__*/_interopDefaultCompat(consola);
|
|
21
|
-
const PQueue__default = /*#__PURE__*/_interopDefaultCompat(PQueue);
|
|
22
21
|
const path__default = /*#__PURE__*/_interopDefaultCompat(path);
|
|
23
22
|
const process__default = /*#__PURE__*/_interopDefaultCompat(process);
|
|
24
|
-
const
|
|
25
|
-
const
|
|
26
|
-
const winston__default = /*#__PURE__*/_interopDefaultCompat(winston);
|
|
23
|
+
const consola__default = /*#__PURE__*/_interopDefaultCompat(consola);
|
|
24
|
+
const PQueue__default = /*#__PURE__*/_interopDefaultCompat(PQueue);
|
|
27
25
|
const cliProgress__default = /*#__PURE__*/_interopDefaultCompat(cliProgress);
|
|
28
26
|
const Options__default = /*#__PURE__*/_interopDefaultCompat(Options);
|
|
29
27
|
const _defaultFormatBar__default = /*#__PURE__*/_interopDefaultCompat(_defaultFormatBar);
|
|
30
28
|
const _defaultFormatTime__default = /*#__PURE__*/_interopDefaultCompat(_defaultFormatTime);
|
|
29
|
+
const fs__default = /*#__PURE__*/_interopDefaultCompat(fs);
|
|
30
|
+
const stripAnsi__default = /*#__PURE__*/_interopDefaultCompat(stripAnsi);
|
|
31
|
+
const winston__default = /*#__PURE__*/_interopDefaultCompat(winston);
|
|
31
32
|
|
|
32
33
|
const siteUrlMap = /* @__PURE__ */ new Map();
|
|
33
34
|
const PQueueMap = /* @__PURE__ */ new Map();
|
|
34
35
|
|
|
36
|
+
const formatter = (options, params, payload) => {
|
|
37
|
+
const { value, total } = params;
|
|
38
|
+
const formatTime = options.formatTime || _defaultFormatTime__default;
|
|
39
|
+
const formatBar = options.formatBar || _defaultFormatBar__default;
|
|
40
|
+
const bar = formatBar(params.progress, options);
|
|
41
|
+
const stopTime = params.stopTime || Date.now();
|
|
42
|
+
const elapsedTime = Math.round((stopTime - params.startTime) / 1e3);
|
|
43
|
+
const duration_formatted = formatTime(elapsedTime, options, 1);
|
|
44
|
+
consola__default.debug(duration_formatted);
|
|
45
|
+
if (!payload.type) {
|
|
46
|
+
options = Object.assign({}, options, cliProgress__default.Presets.rect);
|
|
47
|
+
Options__default.assignDerivedOptions(options);
|
|
48
|
+
const bar2 = formatBar(params.progress, options);
|
|
49
|
+
const { site } = payload;
|
|
50
|
+
const url = payload.url || "";
|
|
51
|
+
return `${bar2} ${value}/${total} | ${utils.colors.yellow(site || "MEODP Item")} | ${utils.colors.cyan(url || "URL")}`;
|
|
52
|
+
} else {
|
|
53
|
+
const { error_count, emoji, url, type } = payload;
|
|
54
|
+
const errorInfo = error_count ? `\u274C${error_count} ` : "";
|
|
55
|
+
return `${bar} \u2705${utils.colors.green(value)}/${total} ${errorInfo}| ${duration_formatted} | ${emoji} ${utils.colors.yellow(type)} | ${utils.colors.cyan(url)}`;
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
const multiBar = new cliProgress__default.MultiBar({
|
|
59
|
+
hideCursor: true,
|
|
60
|
+
// format: '{bar} {percentage}% | ✅{value}/{total} ❌{error_count} | {duration_formatted} | {name}',
|
|
61
|
+
format: formatter,
|
|
62
|
+
barsize: 20,
|
|
63
|
+
autopadding: true,
|
|
64
|
+
barCompleteChar: "\u2588",
|
|
65
|
+
barIncompleteChar: "\u2591",
|
|
66
|
+
// important! redraw everything to avoid "empty" completed bars
|
|
67
|
+
forceRedraw: true
|
|
68
|
+
}, cliProgress__default.Presets.rect);
|
|
69
|
+
const progressBarMap = /* @__PURE__ */ new Map();
|
|
70
|
+
|
|
35
71
|
var __defProp$1 = Object.defineProperty;
|
|
36
72
|
var __defNormalProp$1 = (obj, key, value) => key in obj ? __defProp$1(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
37
73
|
var __publicField$1 = (obj, key, value) => {
|
|
@@ -164,40 +200,93 @@ const COLORFUL_SYMBOLS = {
|
|
|
164
200
|
line: utils.colors.dim(RAW_SYMBOLS.line)
|
|
165
201
|
};
|
|
166
202
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
const
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
203
|
+
function createMEODPLogger() {
|
|
204
|
+
return {
|
|
205
|
+
_log: (...args) => {
|
|
206
|
+
const allLogPath = path__default.resolve(MEODP.logFolder, "all.log");
|
|
207
|
+
LocalLog.log(allLogPath, stripAnsi__default(args.join(" ")));
|
|
208
|
+
},
|
|
209
|
+
log: (...args) => {
|
|
210
|
+
const content = formatArgs(args);
|
|
211
|
+
consola__default.log(content);
|
|
212
|
+
MEODP.logger._log(content);
|
|
213
|
+
},
|
|
214
|
+
inner: (...args) => {
|
|
215
|
+
const content = formatArgs(args);
|
|
216
|
+
consola__default.log(COLORFUL_SYMBOLS.line, " ", content);
|
|
217
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.line, " ", content);
|
|
218
|
+
},
|
|
219
|
+
start: (...args) => {
|
|
220
|
+
const content = formatArgs(args);
|
|
221
|
+
consola__default.start(content);
|
|
222
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.start, content);
|
|
223
|
+
},
|
|
224
|
+
success: (...args) => {
|
|
225
|
+
const content = formatArgs(args);
|
|
226
|
+
consola__default.success(content);
|
|
227
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.success, content);
|
|
228
|
+
},
|
|
229
|
+
info: (...args) => {
|
|
230
|
+
const content = formatArgs(args);
|
|
231
|
+
consola__default.info(content);
|
|
232
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.info, content);
|
|
233
|
+
},
|
|
234
|
+
warn: (...args) => {
|
|
235
|
+
const content = formatArgs(args);
|
|
236
|
+
consola__default.warn(content);
|
|
237
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.warn, content);
|
|
238
|
+
},
|
|
239
|
+
error: (...args) => {
|
|
240
|
+
const content = formatArgs(args);
|
|
241
|
+
consola__default.log(COLORFUL_SYMBOLS.error, content);
|
|
242
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.error, content);
|
|
243
|
+
MEODP.wLogger.error(content);
|
|
244
|
+
}
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function createWinstonLogger() {
|
|
249
|
+
const allLogPath = path__default.resolve(MEODP.logFolder, "all.log");
|
|
250
|
+
const errorLogPath = path__default.resolve(MEODP.logFolder, "error.log");
|
|
251
|
+
const customFormats = [
|
|
252
|
+
winston__default.format.timestamp({
|
|
253
|
+
format: "YYYY-MM-DD HH:mm:ss"
|
|
254
|
+
})
|
|
255
|
+
// winston.format.colorize(),
|
|
256
|
+
// winston.format.errors({ stack: true }),
|
|
257
|
+
// winston.format.json(),
|
|
258
|
+
// winston.format.simple(),
|
|
259
|
+
// winston.format.uncolorize(),
|
|
260
|
+
];
|
|
261
|
+
customFormats.push(
|
|
262
|
+
winston__default.format.printf(({ level, message, timestamp, service }) => {
|
|
263
|
+
const content = [
|
|
264
|
+
timestamp,
|
|
265
|
+
`[${service}]`,
|
|
266
|
+
level.toUpperCase(),
|
|
267
|
+
message
|
|
268
|
+
];
|
|
269
|
+
return content.join(" ");
|
|
270
|
+
})
|
|
271
|
+
);
|
|
272
|
+
const customTransports = [
|
|
273
|
+
new winston__default.transports.File({
|
|
274
|
+
filename: errorLogPath,
|
|
275
|
+
level: "error"
|
|
276
|
+
}),
|
|
277
|
+
new winston__default.transports.File({
|
|
278
|
+
filename: allLogPath
|
|
279
|
+
// level: 'info',
|
|
280
|
+
})
|
|
281
|
+
];
|
|
282
|
+
return winston__default.createLogger({
|
|
283
|
+
level: "info",
|
|
284
|
+
format: winston__default.format.combine(...customFormats),
|
|
285
|
+
defaultMeta: { service: "MEODP" },
|
|
286
|
+
transports: customTransports,
|
|
287
|
+
exitOnError: false
|
|
288
|
+
});
|
|
289
|
+
}
|
|
201
290
|
|
|
202
291
|
function formatArgs(args) {
|
|
203
292
|
return args.join(" ");
|
|
@@ -208,7 +297,7 @@ const TIME_COLOR = {
|
|
|
208
297
|
slow: utils.colors.red
|
|
209
298
|
};
|
|
210
299
|
function getFormattedDuration(duration) {
|
|
211
|
-
const durationText = `${(
|
|
300
|
+
const durationText = `${(duration / 1e3).toString()}s`;
|
|
212
301
|
let type;
|
|
213
302
|
if (duration < 500) {
|
|
214
303
|
type = "fast";
|
|
@@ -225,9 +314,12 @@ function getFormattedDataFromResponse(res) {
|
|
|
225
314
|
const statusInfo = statusText ? `${statusCode?.toString()} ${statusText}` : statusCode?.toString();
|
|
226
315
|
const statusInfoText = statusCode >= 400 ? utils.colors.red(`[${statusInfo}]`) : utils.colors.green(`[${statusInfo}]`);
|
|
227
316
|
const req = res.request();
|
|
228
|
-
const duration = req.timing().responseEnd - req.timing().requestStart;
|
|
317
|
+
const duration = Math.round(req.timing().responseEnd - req.timing().requestStart);
|
|
229
318
|
const durationText = `${COLORFUL_SYMBOLS.line} ${getFormattedDuration(duration)} ${COLORFUL_SYMBOLS.line}`;
|
|
230
319
|
return {
|
|
320
|
+
/**
|
|
321
|
+
* 四舍五入 ms
|
|
322
|
+
*/
|
|
231
323
|
duration,
|
|
232
324
|
durationText,
|
|
233
325
|
statusCode,
|
|
@@ -375,92 +467,62 @@ function getMEODPUrlItemInfo(urlItem) {
|
|
|
375
467
|
};
|
|
376
468
|
}
|
|
377
469
|
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
const content = formatArgs(args);
|
|
406
|
-
consola__default.info(content);
|
|
407
|
-
MEODP.logger._log(COLORFUL_SYMBOLS.info, content);
|
|
408
|
-
},
|
|
409
|
-
warn: (...args) => {
|
|
410
|
-
const content = formatArgs(args);
|
|
411
|
-
consola__default.warn(content);
|
|
412
|
-
MEODP.logger._log(COLORFUL_SYMBOLS.warn, content);
|
|
413
|
-
},
|
|
414
|
-
error: (...args) => {
|
|
415
|
-
const content = formatArgs(args);
|
|
416
|
-
consola__default.log(COLORFUL_SYMBOLS.error, content);
|
|
417
|
-
MEODP.logger._log(COLORFUL_SYMBOLS.error, content);
|
|
418
|
-
MEODP.wLogger.error(content);
|
|
470
|
+
const defaultMEODPConfig = {
|
|
471
|
+
urls: [],
|
|
472
|
+
concurrency: 10,
|
|
473
|
+
log: {
|
|
474
|
+
type: "raw",
|
|
475
|
+
file: true
|
|
476
|
+
}
|
|
477
|
+
};
|
|
478
|
+
function defineConfig(config) {
|
|
479
|
+
return config;
|
|
480
|
+
}
|
|
481
|
+
async function runByConfig(config) {
|
|
482
|
+
const { urls, debug } = config;
|
|
483
|
+
consola__default.level = debug ? 5 : 3;
|
|
484
|
+
const queue = new PQueue__default({
|
|
485
|
+
concurrency: config.concurrency
|
|
486
|
+
});
|
|
487
|
+
if (config.log?.type === "progress") {
|
|
488
|
+
for (const urlItem of urls) {
|
|
489
|
+
const { type, url, emoji } = getMEODPUrlItemInfo(urlItem);
|
|
490
|
+
const bar = multiBar.create(1, 0, {
|
|
491
|
+
emoji,
|
|
492
|
+
url,
|
|
493
|
+
type,
|
|
494
|
+
error_count: 0
|
|
495
|
+
});
|
|
496
|
+
progressBarMap.set(url, bar);
|
|
419
497
|
}
|
|
420
|
-
|
|
498
|
+
const curBar = multiBar.create(1, 0, {
|
|
499
|
+
name: "CURRENT"
|
|
500
|
+
});
|
|
501
|
+
progressBarMap.set("CURRENT", curBar);
|
|
502
|
+
}
|
|
503
|
+
const browser = await getBrowser();
|
|
504
|
+
for (const urlItem of urls) {
|
|
505
|
+
await queue.add(async () => {
|
|
506
|
+
await checkMEODPUrl(urlItem);
|
|
507
|
+
});
|
|
508
|
+
}
|
|
509
|
+
await queue.onIdle();
|
|
510
|
+
if (config.log?.type === "progress") {
|
|
511
|
+
multiBar.stop();
|
|
512
|
+
}
|
|
513
|
+
await browser.close();
|
|
514
|
+
consola__default.debug("browser closed");
|
|
515
|
+
return true;
|
|
421
516
|
}
|
|
422
517
|
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
// winston.format.errors({ stack: true }),
|
|
432
|
-
// winston.format.json(),
|
|
433
|
-
// winston.format.simple(),
|
|
434
|
-
// winston.format.uncolorize(),
|
|
435
|
-
];
|
|
436
|
-
customFormats.push(
|
|
437
|
-
winston__default.format.printf(({ level, message, timestamp, service }) => {
|
|
438
|
-
const content = [
|
|
439
|
-
timestamp,
|
|
440
|
-
`[${service}]`,
|
|
441
|
-
level.toUpperCase(),
|
|
442
|
-
message
|
|
443
|
-
];
|
|
444
|
-
return content.join(" ");
|
|
445
|
-
})
|
|
446
|
-
);
|
|
447
|
-
const customTransports = [
|
|
448
|
-
new winston__default.transports.File({
|
|
449
|
-
filename: errorLogPath,
|
|
450
|
-
level: "error"
|
|
451
|
-
}),
|
|
452
|
-
new winston__default.transports.File({
|
|
453
|
-
filename: allLogPath
|
|
454
|
-
// level: 'info',
|
|
455
|
-
})
|
|
456
|
-
];
|
|
457
|
-
return winston__default.createLogger({
|
|
458
|
-
level: "info",
|
|
459
|
-
format: winston__default.format.combine(...customFormats),
|
|
460
|
-
defaultMeta: { service: "MEODP" },
|
|
461
|
-
transports: customTransports,
|
|
462
|
-
exitOnError: false
|
|
463
|
-
});
|
|
518
|
+
const defaultData = {
|
|
519
|
+
config: defaultMEODPConfig,
|
|
520
|
+
sites: {}
|
|
521
|
+
};
|
|
522
|
+
async function createLowDB(rootDir = process__default.cwd()) {
|
|
523
|
+
const dbJsonPath = path__default.resolve(rootDir, "logs/meodp/db.json");
|
|
524
|
+
const db = await node.JSONFilePreset(dbJsonPath, defaultData);
|
|
525
|
+
return db;
|
|
464
526
|
}
|
|
465
527
|
|
|
466
528
|
var __defProp = Object.defineProperty;
|
|
@@ -485,6 +547,9 @@ const _MEODP = class _MEODP {
|
|
|
485
547
|
await LocalLog.init();
|
|
486
548
|
_MEODP.logFolder = LocalLog.logFolder || path__default.resolve(process__default.cwd(), "logs/meodp");
|
|
487
549
|
_MEODP.wLogger = createWinstonLogger();
|
|
550
|
+
const db = await createLowDB();
|
|
551
|
+
_MEODP.db = db;
|
|
552
|
+
await db.update((data) => data.config = _MEODP.config);
|
|
488
553
|
}
|
|
489
554
|
/**
|
|
490
555
|
* filter link by ignoreLinks
|
|
@@ -533,6 +598,7 @@ __publicField(_MEODP, "browser");
|
|
|
533
598
|
__publicField(_MEODP, "logFolder");
|
|
534
599
|
__publicField(_MEODP, "logger", createMEODPLogger());
|
|
535
600
|
__publicField(_MEODP, "wLogger");
|
|
601
|
+
__publicField(_MEODP, "db");
|
|
536
602
|
let MEODP = _MEODP;
|
|
537
603
|
|
|
538
604
|
async function getBrowser() {
|
|
@@ -581,7 +647,6 @@ async function checkUrlNomoduleAssets(page) {
|
|
|
581
647
|
if (!nomoduleUrl) {
|
|
582
648
|
return;
|
|
583
649
|
}
|
|
584
|
-
const startTime = Date.now();
|
|
585
650
|
const newPage = await context.newPage();
|
|
586
651
|
const response = await newPage.goto(nomoduleUrl);
|
|
587
652
|
if (!response) {
|
|
@@ -592,7 +657,8 @@ async function checkUrlNomoduleAssets(page) {
|
|
|
592
657
|
statusInfo,
|
|
593
658
|
linkText
|
|
594
659
|
} = getFormattedDataFromResponse(response);
|
|
595
|
-
const
|
|
660
|
+
const req = response.request();
|
|
661
|
+
const duration = req.timing().responseEnd - req.timing().requestStart;
|
|
596
662
|
const durationTxt = utils.colors.dim(`\u2502${utils.colors.blue(`${duration.toString().padStart(4, " ")}${utils.colors.white("ms")}`)} \u2502`);
|
|
597
663
|
if (statusCode >= 400) {
|
|
598
664
|
MEODP.logger.inner(" ", COLORFUL_SYMBOLS.error, utils.colors.red(statusInfo), durationTxt, linkText);
|
|
@@ -610,9 +676,31 @@ async function checkSiteUrl(url, options) {
|
|
|
610
676
|
const browser = await getBrowser();
|
|
611
677
|
const page = await browser.newPage();
|
|
612
678
|
const bar = progressBarMap.get(options.urlItem.url);
|
|
613
|
-
const startTime = Date.now();
|
|
614
679
|
const isExternalLink = MEODP.isExternalLink(url, options.urlItem);
|
|
615
680
|
const log = LocalLog.createLog(options.urlItem);
|
|
681
|
+
const siteData = MEODP.db.data.sites[options.urlItem.url];
|
|
682
|
+
const siteLinkItem = {
|
|
683
|
+
url,
|
|
684
|
+
statusCode: 0,
|
|
685
|
+
checkStatus: "pending",
|
|
686
|
+
success: 0,
|
|
687
|
+
failed: 0,
|
|
688
|
+
ignored: 0,
|
|
689
|
+
timeout: 0,
|
|
690
|
+
duration: 0,
|
|
691
|
+
total: 0,
|
|
692
|
+
requests: []
|
|
693
|
+
};
|
|
694
|
+
if (MEODP.isIgnoredLink(url)) {
|
|
695
|
+
siteUrlMap.set(url, {
|
|
696
|
+
statusCode: 0,
|
|
697
|
+
checkStatus: "ignored"
|
|
698
|
+
});
|
|
699
|
+
MEODP.logger.inner(`${utils.colors.dim(`[Ignored] ${utils.colors.underline(url)}`)}`);
|
|
700
|
+
siteData.ignored.push(url);
|
|
701
|
+
await MEODP.db.write();
|
|
702
|
+
return;
|
|
703
|
+
}
|
|
616
704
|
if (!MEODP.config.checkExternalLinks && isExternalLink) {
|
|
617
705
|
let res = null;
|
|
618
706
|
try {
|
|
@@ -620,13 +708,19 @@ async function checkSiteUrl(url, options) {
|
|
|
620
708
|
waitUntil: "load"
|
|
621
709
|
});
|
|
622
710
|
} catch (e) {
|
|
623
|
-
|
|
711
|
+
siteLinkItem.timeout = 1;
|
|
712
|
+
MEODP.logger.inner(
|
|
713
|
+
utils.colors.red("[Timeout]"),
|
|
714
|
+
utils.colors.underline(url)
|
|
715
|
+
);
|
|
716
|
+
consola__default.debug(e);
|
|
624
717
|
}
|
|
625
718
|
if (!res)
|
|
626
719
|
return;
|
|
627
720
|
const {
|
|
628
721
|
statusCode,
|
|
629
722
|
statusInfoText,
|
|
723
|
+
duration,
|
|
630
724
|
durationText
|
|
631
725
|
} = getFormattedDataFromResponse(res);
|
|
632
726
|
const checkStatus = statusCode < 400 ? "passed" : "failed";
|
|
@@ -639,28 +733,59 @@ async function checkSiteUrl(url, options) {
|
|
|
639
733
|
if (statusCode >= 400) {
|
|
640
734
|
MEODP.wLogger.error(` ${statusInfoText} ${url} ${durationText}`);
|
|
641
735
|
}
|
|
736
|
+
siteLinkItem.statusCode = statusCode;
|
|
737
|
+
siteLinkItem.checkStatus = checkStatus;
|
|
738
|
+
siteLinkItem.duration = duration;
|
|
739
|
+
siteLinkItem.total = 1;
|
|
740
|
+
siteLinkItem.success = checkStatus === "passed" ? 1 : 0;
|
|
741
|
+
siteLinkItem.failed = checkStatus === "failed" ? 1 : 0;
|
|
742
|
+
siteLinkItem.requests = [];
|
|
743
|
+
siteData.links.push(siteLinkItem);
|
|
744
|
+
await MEODP.db.write();
|
|
642
745
|
return;
|
|
643
746
|
}
|
|
644
747
|
const urlMap = registerPageEvents(page, options.urlItem);
|
|
645
|
-
const { success, failed, total, ignored, timeout } = parseUrlMap(urlMap);
|
|
646
748
|
try {
|
|
647
749
|
const res = await page.goto(url, {
|
|
648
750
|
waitUntil: "networkidle"
|
|
649
751
|
});
|
|
650
752
|
const statusCode = res?.status() || 0;
|
|
753
|
+
const statusText = res?.statusText();
|
|
754
|
+
const req = res?.request();
|
|
755
|
+
const duration = Math.round((req?.timing()?.responseEnd || 0) - (req?.timing()?.requestStart || 0));
|
|
651
756
|
siteUrlMap.set(url, {
|
|
652
757
|
response: res,
|
|
653
758
|
statusCode,
|
|
654
759
|
checkStatus: "goto"
|
|
655
760
|
});
|
|
656
|
-
const
|
|
657
|
-
|
|
761
|
+
const { success, failed, total, ignored, timeout } = parseUrlMap(urlMap);
|
|
762
|
+
siteLinkItem.title = await page.title();
|
|
763
|
+
siteLinkItem.statusCode = statusCode;
|
|
764
|
+
siteLinkItem.statusText = statusText;
|
|
765
|
+
siteLinkItem.checkStatus = "goto";
|
|
766
|
+
siteLinkItem.duration = duration;
|
|
767
|
+
siteLinkItem.total = total.count;
|
|
768
|
+
siteLinkItem.success = success.count;
|
|
769
|
+
siteLinkItem.failed = failed.count;
|
|
770
|
+
siteLinkItem.timeout = timeout.count;
|
|
771
|
+
siteLinkItem.ignored = ignored.count;
|
|
772
|
+
siteLinkItem.requests = Array.from(urlMap.values()).map((value) => {
|
|
773
|
+
const res2 = value.response;
|
|
774
|
+
return {
|
|
775
|
+
failed: value.failed,
|
|
776
|
+
ignored: value.ignored,
|
|
777
|
+
// res.url 还没用返回
|
|
778
|
+
url: req?.url(),
|
|
779
|
+
statusCode: res2?.status(),
|
|
780
|
+
statusText: res2?.statusText()
|
|
781
|
+
};
|
|
782
|
+
});
|
|
658
783
|
const logInfo = [
|
|
659
784
|
COLORFUL_SYMBOLS.line,
|
|
660
785
|
" ",
|
|
661
786
|
utils.colors.green(`[${statusCode}${statusText ? ` ${statusText}` : ""}]`),
|
|
662
787
|
utils.colors.cyan(url),
|
|
663
|
-
utils.colors.dim(`(in ${duration}
|
|
788
|
+
utils.colors.dim(`(in ${duration}ms)`),
|
|
664
789
|
total.text,
|
|
665
790
|
success.text,
|
|
666
791
|
failed.text,
|
|
@@ -686,23 +811,25 @@ async function checkSiteUrl(url, options) {
|
|
|
686
811
|
if (!info.response && !info.ignored) {
|
|
687
812
|
MEODP.logger.log(COLORFUL_SYMBOLS.line, " ", COLORFUL_SYMBOLS.error, utils.colors.red("Timeout:"), utils.colors.underline(url2));
|
|
688
813
|
MEODP.wLogger.error(` Timeout: ${url2}`);
|
|
814
|
+
siteLinkItem.checkStatus = "timeout";
|
|
689
815
|
}
|
|
690
816
|
}
|
|
691
817
|
let noModuleAssetsPassed = true;
|
|
692
818
|
if (options.checkNoModule) {
|
|
693
819
|
noModuleAssetsPassed = await checkUrlNomoduleAssets(page);
|
|
694
820
|
}
|
|
695
|
-
const urlMapValues = Array.from(urlMap.values());
|
|
696
821
|
const siteUrlItem = siteUrlMap.get(url);
|
|
697
822
|
if (siteUrlItem) {
|
|
698
|
-
|
|
823
|
+
const isAllRequestsPassed = siteLinkItem.requests.filter((value) => !value.ignored).every((value) => value.statusCode && value.statusCode < 400);
|
|
824
|
+
if (isAllRequestsPassed && noModuleAssetsPassed) {
|
|
699
825
|
siteUrlItem.checkStatus = "passed";
|
|
700
|
-
const successCount = Array.from(siteUrlMap.values()).filter((value) => value.checkStatus === "passed").length;
|
|
701
|
-
bar?.update(successCount);
|
|
702
826
|
} else {
|
|
703
827
|
siteUrlItem.checkStatus = "failed";
|
|
704
828
|
}
|
|
829
|
+
siteLinkItem.checkStatus = siteUrlItem.checkStatus;
|
|
705
830
|
}
|
|
831
|
+
siteData.links.push(siteLinkItem);
|
|
832
|
+
await MEODP.db.write();
|
|
706
833
|
if (!isExternalLink) {
|
|
707
834
|
const queue = PQueueMap.get(url);
|
|
708
835
|
if (queue) {
|
|
@@ -723,6 +850,7 @@ async function checkSiteUrl(url, options) {
|
|
|
723
850
|
}
|
|
724
851
|
}
|
|
725
852
|
await page.close();
|
|
853
|
+
return siteLinkItem;
|
|
726
854
|
}
|
|
727
855
|
async function checkSite(props) {
|
|
728
856
|
const { url } = props;
|
|
@@ -739,11 +867,25 @@ async function checkSite(props) {
|
|
|
739
867
|
statusCode: 0,
|
|
740
868
|
checkStatus: "pending"
|
|
741
869
|
});
|
|
870
|
+
await MEODP.db.update((data) => {
|
|
871
|
+
data.sites[homeUrl] = {
|
|
872
|
+
name: props.name,
|
|
873
|
+
url: homeUrl,
|
|
874
|
+
checkStatus: "pending",
|
|
875
|
+
links: [],
|
|
876
|
+
ignored: []
|
|
877
|
+
};
|
|
878
|
+
});
|
|
742
879
|
bar?.setTotal(siteUrlMap.size);
|
|
743
|
-
await checkSiteUrl(homeUrl, {
|
|
880
|
+
const siteLinkItem = await checkSiteUrl(homeUrl, {
|
|
744
881
|
urlItem: props,
|
|
745
882
|
checkNoModule: true
|
|
746
883
|
});
|
|
884
|
+
await MEODP.db.update((data) => {
|
|
885
|
+
const siteItem = data.sites[homeUrl];
|
|
886
|
+
siteItem.title = siteLinkItem?.title;
|
|
887
|
+
siteItem.checkStatus = siteItem.links.every((link) => link.checkStatus === "passed") ? "passed" : "failed";
|
|
888
|
+
});
|
|
747
889
|
await queue.onIdle();
|
|
748
890
|
await page.close();
|
|
749
891
|
await context.close();
|
|
@@ -816,54 +958,6 @@ async function checkMEODPUrl(meodpUrl) {
|
|
|
816
958
|
MEODP.logger.success(`\u{1F577}\uFE0F ${utils.colors.greenBright("[DONE]")} ${utils.colors.gray("in")} ${duration / 1e3}s.`);
|
|
817
959
|
}
|
|
818
960
|
|
|
819
|
-
const defaultMEODPConfig = {
|
|
820
|
-
urls: [],
|
|
821
|
-
concurrency: 10,
|
|
822
|
-
log: {
|
|
823
|
-
type: "raw",
|
|
824
|
-
file: true
|
|
825
|
-
}
|
|
826
|
-
};
|
|
827
|
-
function defineConfig(config) {
|
|
828
|
-
return config;
|
|
829
|
-
}
|
|
830
|
-
async function runByConfig(config) {
|
|
831
|
-
const { urls, debug } = config;
|
|
832
|
-
consola__default.level = debug ? 5 : 3;
|
|
833
|
-
const queue = new PQueue__default({
|
|
834
|
-
concurrency: config.concurrency
|
|
835
|
-
});
|
|
836
|
-
if (config.log?.type === "progress") {
|
|
837
|
-
for (const urlItem of urls) {
|
|
838
|
-
const { type, url, emoji } = getMEODPUrlItemInfo(urlItem);
|
|
839
|
-
const bar = multiBar.create(1, 0, {
|
|
840
|
-
emoji,
|
|
841
|
-
url,
|
|
842
|
-
type,
|
|
843
|
-
error_count: 0
|
|
844
|
-
});
|
|
845
|
-
progressBarMap.set(url, bar);
|
|
846
|
-
}
|
|
847
|
-
const curBar = multiBar.create(1, 0, {
|
|
848
|
-
name: "CURRENT"
|
|
849
|
-
});
|
|
850
|
-
progressBarMap.set("CURRENT", curBar);
|
|
851
|
-
}
|
|
852
|
-
const browser = await getBrowser();
|
|
853
|
-
for (const urlItem of urls) {
|
|
854
|
-
await queue.add(async () => {
|
|
855
|
-
await checkMEODPUrl(urlItem);
|
|
856
|
-
});
|
|
857
|
-
}
|
|
858
|
-
await queue.onIdle();
|
|
859
|
-
if (config.log?.type === "progress") {
|
|
860
|
-
multiBar.stop();
|
|
861
|
-
}
|
|
862
|
-
await browser.close();
|
|
863
|
-
consola__default.debug("browser closed");
|
|
864
|
-
return true;
|
|
865
|
-
}
|
|
866
|
-
|
|
867
961
|
exports.LocalLog = LocalLog;
|
|
868
962
|
exports.MEODP = MEODP;
|
|
869
963
|
exports.PQueueMap = PQueueMap;
|
|
@@ -872,6 +966,7 @@ exports.checkMEODPUrl = checkMEODPUrl;
|
|
|
872
966
|
exports.checkSite = checkSite;
|
|
873
967
|
exports.checkSiteUrl = checkSiteUrl;
|
|
874
968
|
exports.checkUrlNomoduleAssets = checkUrlNomoduleAssets;
|
|
969
|
+
exports.createLowDB = createLowDB;
|
|
875
970
|
exports.createMEODPLogger = createMEODPLogger;
|
|
876
971
|
exports.createWinstonLogger = createWinstonLogger;
|
|
877
972
|
exports.defaultMEODPConfig = defaultMEODPConfig;
|