meodp 0.0.6 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bin/index.mjs +0 -0
- package/dist/cli/index.cjs +168 -18
- package/dist/cli/index.mjs +154 -7
- package/dist/index.cjs +39 -37
- package/dist/index.d.cts +87 -3
- package/dist/index.d.mts +87 -3
- package/dist/index.d.ts +87 -3
- package/dist/index.mjs +9 -8
- package/dist/shared/{meodp.9c5df07b.mjs → meodp.5489574b.mjs} +275 -186
- package/dist/shared/{meodp.c2959ba5.cjs → meodp.9bf8b8ff.cjs} +280 -190
- package/package.json +6 -2
|
@@ -1,37 +1,73 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
const consola = require('consola');
|
|
4
|
-
const PQueue = require('p-queue');
|
|
5
3
|
const utils = require('consola/utils');
|
|
6
4
|
const playwright = require('playwright');
|
|
7
5
|
const path = require('node:path');
|
|
8
6
|
const process = require('node:process');
|
|
9
|
-
const
|
|
10
|
-
const
|
|
11
|
-
const
|
|
12
|
-
const winston = require('winston');
|
|
7
|
+
const consola = require('consola');
|
|
8
|
+
const node = require('lowdb/node');
|
|
9
|
+
const PQueue = require('p-queue');
|
|
13
10
|
const cliProgress = require('cli-progress');
|
|
14
11
|
const Options = require('cli-progress/lib/options.js');
|
|
15
12
|
const _defaultFormatBar = require('cli-progress/lib/format-bar.js');
|
|
16
13
|
const _defaultFormatTime = require('cli-progress/lib/format-time.js');
|
|
14
|
+
const dateFns = require('date-fns');
|
|
15
|
+
const fs = require('fs-extra');
|
|
16
|
+
const stripAnsi = require('strip-ansi');
|
|
17
|
+
const winston = require('winston');
|
|
17
18
|
|
|
18
19
|
function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e.default : e; }
|
|
19
20
|
|
|
20
|
-
const consola__default = /*#__PURE__*/_interopDefaultCompat(consola);
|
|
21
|
-
const PQueue__default = /*#__PURE__*/_interopDefaultCompat(PQueue);
|
|
22
21
|
const path__default = /*#__PURE__*/_interopDefaultCompat(path);
|
|
23
22
|
const process__default = /*#__PURE__*/_interopDefaultCompat(process);
|
|
24
|
-
const
|
|
25
|
-
const
|
|
26
|
-
const winston__default = /*#__PURE__*/_interopDefaultCompat(winston);
|
|
23
|
+
const consola__default = /*#__PURE__*/_interopDefaultCompat(consola);
|
|
24
|
+
const PQueue__default = /*#__PURE__*/_interopDefaultCompat(PQueue);
|
|
27
25
|
const cliProgress__default = /*#__PURE__*/_interopDefaultCompat(cliProgress);
|
|
28
26
|
const Options__default = /*#__PURE__*/_interopDefaultCompat(Options);
|
|
29
27
|
const _defaultFormatBar__default = /*#__PURE__*/_interopDefaultCompat(_defaultFormatBar);
|
|
30
28
|
const _defaultFormatTime__default = /*#__PURE__*/_interopDefaultCompat(_defaultFormatTime);
|
|
29
|
+
const fs__default = /*#__PURE__*/_interopDefaultCompat(fs);
|
|
30
|
+
const stripAnsi__default = /*#__PURE__*/_interopDefaultCompat(stripAnsi);
|
|
31
|
+
const winston__default = /*#__PURE__*/_interopDefaultCompat(winston);
|
|
31
32
|
|
|
32
33
|
const siteUrlMap = /* @__PURE__ */ new Map();
|
|
33
34
|
const PQueueMap = /* @__PURE__ */ new Map();
|
|
34
35
|
|
|
36
|
+
const formatter = (options, params, payload) => {
|
|
37
|
+
const { value, total } = params;
|
|
38
|
+
const formatTime = options.formatTime || _defaultFormatTime__default;
|
|
39
|
+
const formatBar = options.formatBar || _defaultFormatBar__default;
|
|
40
|
+
const bar = formatBar(params.progress, options);
|
|
41
|
+
const stopTime = params.stopTime || Date.now();
|
|
42
|
+
const elapsedTime = Math.round((stopTime - params.startTime) / 1e3);
|
|
43
|
+
const duration_formatted = formatTime(elapsedTime, options, 1);
|
|
44
|
+
consola__default.debug(duration_formatted);
|
|
45
|
+
if (!payload.type) {
|
|
46
|
+
options = Object.assign({}, options, cliProgress__default.Presets.rect);
|
|
47
|
+
Options__default.assignDerivedOptions(options);
|
|
48
|
+
const bar2 = formatBar(params.progress, options);
|
|
49
|
+
const { site } = payload;
|
|
50
|
+
const url = payload.url || "";
|
|
51
|
+
return `${bar2} ${value}/${total} | ${utils.colors.yellow(site || "MEODP Item")} | ${utils.colors.cyan(url || "URL")}`;
|
|
52
|
+
} else {
|
|
53
|
+
const { error_count, emoji, url, type } = payload;
|
|
54
|
+
const errorInfo = error_count ? `\u274C${error_count} ` : "";
|
|
55
|
+
return `${bar} \u2705${utils.colors.green(value)}/${total} ${errorInfo}| ${duration_formatted} | ${emoji} ${utils.colors.yellow(type)} | ${utils.colors.cyan(url)}`;
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
const multiBar = new cliProgress__default.MultiBar({
|
|
59
|
+
hideCursor: true,
|
|
60
|
+
// format: '{bar} {percentage}% | ✅{value}/{total} ❌{error_count} | {duration_formatted} | {name}',
|
|
61
|
+
format: formatter,
|
|
62
|
+
barsize: 20,
|
|
63
|
+
autopadding: true,
|
|
64
|
+
barCompleteChar: "\u2588",
|
|
65
|
+
barIncompleteChar: "\u2591",
|
|
66
|
+
// important! redraw everything to avoid "empty" completed bars
|
|
67
|
+
forceRedraw: true
|
|
68
|
+
}, cliProgress__default.Presets.rect);
|
|
69
|
+
const progressBarMap = /* @__PURE__ */ new Map();
|
|
70
|
+
|
|
35
71
|
var __defProp$1 = Object.defineProperty;
|
|
36
72
|
var __defNormalProp$1 = (obj, key, value) => key in obj ? __defProp$1(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
37
73
|
var __publicField$1 = (obj, key, value) => {
|
|
@@ -164,40 +200,93 @@ const COLORFUL_SYMBOLS = {
|
|
|
164
200
|
line: utils.colors.dim(RAW_SYMBOLS.line)
|
|
165
201
|
};
|
|
166
202
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
const
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
203
|
+
function createMEODPLogger() {
|
|
204
|
+
return {
|
|
205
|
+
_log: (...args) => {
|
|
206
|
+
const allLogPath = path__default.resolve(MEODP.logFolder, "all.log");
|
|
207
|
+
LocalLog.log(allLogPath, stripAnsi__default(args.join(" ")));
|
|
208
|
+
},
|
|
209
|
+
log: (...args) => {
|
|
210
|
+
const content = formatArgs(args);
|
|
211
|
+
consola__default.log(content);
|
|
212
|
+
MEODP.logger._log(content);
|
|
213
|
+
},
|
|
214
|
+
inner: (...args) => {
|
|
215
|
+
const content = formatArgs(args);
|
|
216
|
+
consola__default.log(COLORFUL_SYMBOLS.line, " ", content);
|
|
217
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.line, " ", content);
|
|
218
|
+
},
|
|
219
|
+
start: (...args) => {
|
|
220
|
+
const content = formatArgs(args);
|
|
221
|
+
consola__default.start(content);
|
|
222
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.start, content);
|
|
223
|
+
},
|
|
224
|
+
success: (...args) => {
|
|
225
|
+
const content = formatArgs(args);
|
|
226
|
+
consola__default.success(content);
|
|
227
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.success, content);
|
|
228
|
+
},
|
|
229
|
+
info: (...args) => {
|
|
230
|
+
const content = formatArgs(args);
|
|
231
|
+
consola__default.info(content);
|
|
232
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.info, content);
|
|
233
|
+
},
|
|
234
|
+
warn: (...args) => {
|
|
235
|
+
const content = formatArgs(args);
|
|
236
|
+
consola__default.warn(content);
|
|
237
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.warn, content);
|
|
238
|
+
},
|
|
239
|
+
error: (...args) => {
|
|
240
|
+
const content = formatArgs(args);
|
|
241
|
+
consola__default.log(COLORFUL_SYMBOLS.error, content);
|
|
242
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.error, content);
|
|
243
|
+
MEODP.wLogger.error(content);
|
|
244
|
+
}
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function createWinstonLogger() {
|
|
249
|
+
const allLogPath = path__default.resolve(MEODP.logFolder, "all.log");
|
|
250
|
+
const errorLogPath = path__default.resolve(MEODP.logFolder, "error.log");
|
|
251
|
+
const customFormats = [
|
|
252
|
+
winston__default.format.timestamp({
|
|
253
|
+
format: "YYYY-MM-DD HH:mm:ss"
|
|
254
|
+
})
|
|
255
|
+
// winston.format.colorize(),
|
|
256
|
+
// winston.format.errors({ stack: true }),
|
|
257
|
+
// winston.format.json(),
|
|
258
|
+
// winston.format.simple(),
|
|
259
|
+
// winston.format.uncolorize(),
|
|
260
|
+
];
|
|
261
|
+
customFormats.push(
|
|
262
|
+
winston__default.format.printf(({ level, message, timestamp, service }) => {
|
|
263
|
+
const content = [
|
|
264
|
+
timestamp,
|
|
265
|
+
`[${service}]`,
|
|
266
|
+
level.toUpperCase(),
|
|
267
|
+
message
|
|
268
|
+
];
|
|
269
|
+
return content.join(" ");
|
|
270
|
+
})
|
|
271
|
+
);
|
|
272
|
+
const customTransports = [
|
|
273
|
+
new winston__default.transports.File({
|
|
274
|
+
filename: errorLogPath,
|
|
275
|
+
level: "error"
|
|
276
|
+
}),
|
|
277
|
+
new winston__default.transports.File({
|
|
278
|
+
filename: allLogPath
|
|
279
|
+
// level: 'info',
|
|
280
|
+
})
|
|
281
|
+
];
|
|
282
|
+
return winston__default.createLogger({
|
|
283
|
+
level: "info",
|
|
284
|
+
format: winston__default.format.combine(...customFormats),
|
|
285
|
+
defaultMeta: { service: "MEODP" },
|
|
286
|
+
transports: customTransports,
|
|
287
|
+
exitOnError: false
|
|
288
|
+
});
|
|
289
|
+
}
|
|
201
290
|
|
|
202
291
|
function formatArgs(args) {
|
|
203
292
|
return args.join(" ");
|
|
@@ -208,7 +297,7 @@ const TIME_COLOR = {
|
|
|
208
297
|
slow: utils.colors.red
|
|
209
298
|
};
|
|
210
299
|
function getFormattedDuration(duration) {
|
|
211
|
-
const durationText = `${(
|
|
300
|
+
const durationText = `${(duration / 1e3).toString()}s`;
|
|
212
301
|
let type;
|
|
213
302
|
if (duration < 500) {
|
|
214
303
|
type = "fast";
|
|
@@ -225,9 +314,12 @@ function getFormattedDataFromResponse(res) {
|
|
|
225
314
|
const statusInfo = statusText ? `${statusCode?.toString()} ${statusText}` : statusCode?.toString();
|
|
226
315
|
const statusInfoText = statusCode >= 400 ? utils.colors.red(`[${statusInfo}]`) : utils.colors.green(`[${statusInfo}]`);
|
|
227
316
|
const req = res.request();
|
|
228
|
-
const duration = req.timing().responseEnd - req.timing().requestStart;
|
|
317
|
+
const duration = Math.round(req.timing().responseEnd - req.timing().requestStart);
|
|
229
318
|
const durationText = `${COLORFUL_SYMBOLS.line} ${getFormattedDuration(duration)} ${COLORFUL_SYMBOLS.line}`;
|
|
230
319
|
return {
|
|
320
|
+
/**
|
|
321
|
+
* 四舍五入 ms
|
|
322
|
+
*/
|
|
231
323
|
duration,
|
|
232
324
|
durationText,
|
|
233
325
|
statusCode,
|
|
@@ -375,92 +467,62 @@ function getMEODPUrlItemInfo(urlItem) {
|
|
|
375
467
|
};
|
|
376
468
|
}
|
|
377
469
|
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
const content = formatArgs(args);
|
|
406
|
-
consola__default.info(content);
|
|
407
|
-
MEODP.logger._log(COLORFUL_SYMBOLS.info, content);
|
|
408
|
-
},
|
|
409
|
-
warn: (...args) => {
|
|
410
|
-
const content = formatArgs(args);
|
|
411
|
-
consola__default.warn(content);
|
|
412
|
-
MEODP.logger._log(COLORFUL_SYMBOLS.warn, content);
|
|
413
|
-
},
|
|
414
|
-
error: (...args) => {
|
|
415
|
-
const content = formatArgs(args);
|
|
416
|
-
consola__default.log(COLORFUL_SYMBOLS.error, content);
|
|
417
|
-
MEODP.logger._log(COLORFUL_SYMBOLS.error, content);
|
|
418
|
-
MEODP.wLogger.error(content);
|
|
470
|
+
const defaultMEODPConfig = {
|
|
471
|
+
urls: [],
|
|
472
|
+
concurrency: 10,
|
|
473
|
+
log: {
|
|
474
|
+
type: "raw",
|
|
475
|
+
file: true
|
|
476
|
+
}
|
|
477
|
+
};
|
|
478
|
+
function defineConfig(config) {
|
|
479
|
+
return config;
|
|
480
|
+
}
|
|
481
|
+
async function runByConfig(config) {
|
|
482
|
+
const { urls, debug } = config;
|
|
483
|
+
consola__default.level = debug ? 5 : 3;
|
|
484
|
+
const queue = new PQueue__default({
|
|
485
|
+
concurrency: config.concurrency
|
|
486
|
+
});
|
|
487
|
+
if (config.log?.type === "progress") {
|
|
488
|
+
for (const urlItem of urls) {
|
|
489
|
+
const { type, url, emoji } = getMEODPUrlItemInfo(urlItem);
|
|
490
|
+
const bar = multiBar.create(1, 0, {
|
|
491
|
+
emoji,
|
|
492
|
+
url,
|
|
493
|
+
type,
|
|
494
|
+
error_count: 0
|
|
495
|
+
});
|
|
496
|
+
progressBarMap.set(url, bar);
|
|
419
497
|
}
|
|
420
|
-
|
|
498
|
+
const curBar = multiBar.create(1, 0, {
|
|
499
|
+
name: "CURRENT"
|
|
500
|
+
});
|
|
501
|
+
progressBarMap.set("CURRENT", curBar);
|
|
502
|
+
}
|
|
503
|
+
const browser = await getBrowser();
|
|
504
|
+
for (const urlItem of urls) {
|
|
505
|
+
await queue.add(async () => {
|
|
506
|
+
await checkMEODPUrl(urlItem);
|
|
507
|
+
});
|
|
508
|
+
}
|
|
509
|
+
await queue.onIdle();
|
|
510
|
+
if (config.log?.type === "progress") {
|
|
511
|
+
multiBar.stop();
|
|
512
|
+
}
|
|
513
|
+
await browser.close();
|
|
514
|
+
consola__default.debug("browser closed");
|
|
515
|
+
return true;
|
|
421
516
|
}
|
|
422
517
|
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
// winston.format.errors({ stack: true }),
|
|
432
|
-
// winston.format.json(),
|
|
433
|
-
// winston.format.simple(),
|
|
434
|
-
// winston.format.uncolorize(),
|
|
435
|
-
];
|
|
436
|
-
customFormats.push(
|
|
437
|
-
winston__default.format.printf(({ level, message, timestamp, service }) => {
|
|
438
|
-
const content = [
|
|
439
|
-
timestamp,
|
|
440
|
-
`[${service}]`,
|
|
441
|
-
level.toUpperCase(),
|
|
442
|
-
message
|
|
443
|
-
];
|
|
444
|
-
return content.join(" ");
|
|
445
|
-
})
|
|
446
|
-
);
|
|
447
|
-
const customTransports = [
|
|
448
|
-
new winston__default.transports.File({
|
|
449
|
-
filename: errorLogPath,
|
|
450
|
-
level: "error"
|
|
451
|
-
}),
|
|
452
|
-
new winston__default.transports.File({
|
|
453
|
-
filename: allLogPath
|
|
454
|
-
// level: 'info',
|
|
455
|
-
})
|
|
456
|
-
];
|
|
457
|
-
return winston__default.createLogger({
|
|
458
|
-
level: "info",
|
|
459
|
-
format: winston__default.format.combine(...customFormats),
|
|
460
|
-
defaultMeta: { service: "MEODP" },
|
|
461
|
-
transports: customTransports,
|
|
462
|
-
exitOnError: false
|
|
463
|
-
});
|
|
518
|
+
const defaultData = {
|
|
519
|
+
config: defaultMEODPConfig,
|
|
520
|
+
sites: {}
|
|
521
|
+
};
|
|
522
|
+
async function createLowDB(rootDir = process__default.cwd()) {
|
|
523
|
+
const dbJsonPath = path__default.resolve(rootDir, "logs/meodp/db.json");
|
|
524
|
+
const db = await node.JSONFilePreset(dbJsonPath, defaultData);
|
|
525
|
+
return db;
|
|
464
526
|
}
|
|
465
527
|
|
|
466
528
|
var __defProp = Object.defineProperty;
|
|
@@ -485,6 +547,9 @@ const _MEODP = class _MEODP {
|
|
|
485
547
|
await LocalLog.init();
|
|
486
548
|
_MEODP.logFolder = LocalLog.logFolder || path__default.resolve(process__default.cwd(), "logs/meodp");
|
|
487
549
|
_MEODP.wLogger = createWinstonLogger();
|
|
550
|
+
const db = await createLowDB();
|
|
551
|
+
_MEODP.db = db;
|
|
552
|
+
await db.update((data) => data.config = _MEODP.config);
|
|
488
553
|
}
|
|
489
554
|
/**
|
|
490
555
|
* filter link by ignoreLinks
|
|
@@ -533,6 +598,7 @@ __publicField(_MEODP, "browser");
|
|
|
533
598
|
__publicField(_MEODP, "logFolder");
|
|
534
599
|
__publicField(_MEODP, "logger", createMEODPLogger());
|
|
535
600
|
__publicField(_MEODP, "wLogger");
|
|
601
|
+
__publicField(_MEODP, "db");
|
|
536
602
|
let MEODP = _MEODP;
|
|
537
603
|
|
|
538
604
|
async function getBrowser() {
|
|
@@ -581,7 +647,6 @@ async function checkUrlNomoduleAssets(page) {
|
|
|
581
647
|
if (!nomoduleUrl) {
|
|
582
648
|
return;
|
|
583
649
|
}
|
|
584
|
-
const startTime = Date.now();
|
|
585
650
|
const newPage = await context.newPage();
|
|
586
651
|
const response = await newPage.goto(nomoduleUrl);
|
|
587
652
|
if (!response) {
|
|
@@ -592,7 +657,8 @@ async function checkUrlNomoduleAssets(page) {
|
|
|
592
657
|
statusInfo,
|
|
593
658
|
linkText
|
|
594
659
|
} = getFormattedDataFromResponse(response);
|
|
595
|
-
const
|
|
660
|
+
const req = response.request();
|
|
661
|
+
const duration = req.timing().responseEnd - req.timing().requestStart;
|
|
596
662
|
const durationTxt = utils.colors.dim(`\u2502${utils.colors.blue(`${duration.toString().padStart(4, " ")}${utils.colors.white("ms")}`)} \u2502`);
|
|
597
663
|
if (statusCode >= 400) {
|
|
598
664
|
MEODP.logger.inner(" ", COLORFUL_SYMBOLS.error, utils.colors.red(statusInfo), durationTxt, linkText);
|
|
@@ -610,9 +676,31 @@ async function checkSiteUrl(url, options) {
|
|
|
610
676
|
const browser = await getBrowser();
|
|
611
677
|
const page = await browser.newPage();
|
|
612
678
|
const bar = progressBarMap.get(options.urlItem.url);
|
|
613
|
-
const startTime = Date.now();
|
|
614
679
|
const isExternalLink = MEODP.isExternalLink(url, options.urlItem);
|
|
615
680
|
const log = LocalLog.createLog(options.urlItem);
|
|
681
|
+
const siteData = MEODP.db.data.sites[options.urlItem.url];
|
|
682
|
+
const siteLinkItem = {
|
|
683
|
+
url,
|
|
684
|
+
statusCode: 0,
|
|
685
|
+
checkStatus: "pending",
|
|
686
|
+
success: 0,
|
|
687
|
+
failed: 0,
|
|
688
|
+
ignored: 0,
|
|
689
|
+
timeout: 0,
|
|
690
|
+
duration: 0,
|
|
691
|
+
total: 0,
|
|
692
|
+
requests: []
|
|
693
|
+
};
|
|
694
|
+
if (MEODP.isIgnoredLink(url)) {
|
|
695
|
+
siteUrlMap.set(url, {
|
|
696
|
+
statusCode: 0,
|
|
697
|
+
checkStatus: "ignored"
|
|
698
|
+
});
|
|
699
|
+
MEODP.logger.inner(`${utils.colors.dim(`[Ignored] ${utils.colors.underline(url)}`)}`);
|
|
700
|
+
siteData.ignored.push(url);
|
|
701
|
+
await MEODP.db.write();
|
|
702
|
+
return;
|
|
703
|
+
}
|
|
616
704
|
if (!MEODP.config.checkExternalLinks && isExternalLink) {
|
|
617
705
|
let res = null;
|
|
618
706
|
try {
|
|
@@ -620,6 +708,7 @@ async function checkSiteUrl(url, options) {
|
|
|
620
708
|
waitUntil: "load"
|
|
621
709
|
});
|
|
622
710
|
} catch (e) {
|
|
711
|
+
siteLinkItem.timeout = 1;
|
|
623
712
|
MEODP.logger.error(e);
|
|
624
713
|
}
|
|
625
714
|
if (!res)
|
|
@@ -627,6 +716,7 @@ async function checkSiteUrl(url, options) {
|
|
|
627
716
|
const {
|
|
628
717
|
statusCode,
|
|
629
718
|
statusInfoText,
|
|
719
|
+
duration,
|
|
630
720
|
durationText
|
|
631
721
|
} = getFormattedDataFromResponse(res);
|
|
632
722
|
const checkStatus = statusCode < 400 ? "passed" : "failed";
|
|
@@ -639,28 +729,58 @@ async function checkSiteUrl(url, options) {
|
|
|
639
729
|
if (statusCode >= 400) {
|
|
640
730
|
MEODP.wLogger.error(` ${statusInfoText} ${url} ${durationText}`);
|
|
641
731
|
}
|
|
732
|
+
siteLinkItem.statusCode = statusCode;
|
|
733
|
+
siteLinkItem.checkStatus = checkStatus;
|
|
734
|
+
siteLinkItem.duration = duration;
|
|
735
|
+
siteLinkItem.total = 1;
|
|
736
|
+
siteLinkItem.success = checkStatus === "passed" ? 1 : 0;
|
|
737
|
+
siteLinkItem.failed = checkStatus === "failed" ? 1 : 0;
|
|
738
|
+
siteLinkItem.requests = [];
|
|
739
|
+
siteData.links.push(siteLinkItem);
|
|
740
|
+
await MEODP.db.write();
|
|
642
741
|
return;
|
|
643
742
|
}
|
|
644
743
|
const urlMap = registerPageEvents(page, options.urlItem);
|
|
645
|
-
const { success, failed, total, ignored, timeout } = parseUrlMap(urlMap);
|
|
646
744
|
try {
|
|
647
745
|
const res = await page.goto(url, {
|
|
648
746
|
waitUntil: "networkidle"
|
|
649
747
|
});
|
|
650
748
|
const statusCode = res?.status() || 0;
|
|
749
|
+
const statusText = res?.statusText();
|
|
750
|
+
const req = res?.request();
|
|
751
|
+
const duration = Math.round((req?.timing()?.responseEnd || 0) - (req?.timing()?.requestStart || 0));
|
|
651
752
|
siteUrlMap.set(url, {
|
|
652
753
|
response: res,
|
|
653
754
|
statusCode,
|
|
654
755
|
checkStatus: "goto"
|
|
655
756
|
});
|
|
656
|
-
const
|
|
657
|
-
|
|
757
|
+
const { success, failed, total, ignored, timeout } = parseUrlMap(urlMap);
|
|
758
|
+
siteLinkItem.title = await page.title();
|
|
759
|
+
siteLinkItem.statusCode = statusCode;
|
|
760
|
+
siteLinkItem.statusText = statusText;
|
|
761
|
+
siteLinkItem.checkStatus = "goto";
|
|
762
|
+
siteLinkItem.duration = duration;
|
|
763
|
+
siteLinkItem.total = total.count;
|
|
764
|
+
siteLinkItem.success = success.count;
|
|
765
|
+
siteLinkItem.failed = failed.count;
|
|
766
|
+
siteLinkItem.timeout = timeout.count;
|
|
767
|
+
siteLinkItem.ignored = ignored.count;
|
|
768
|
+
siteLinkItem.requests = Array.from(urlMap.values()).map((value) => {
|
|
769
|
+
const res2 = value.response;
|
|
770
|
+
return {
|
|
771
|
+
failed: value.failed,
|
|
772
|
+
ignored: value.ignored,
|
|
773
|
+
url: res2?.url(),
|
|
774
|
+
statusCode: res2?.status(),
|
|
775
|
+
statusText: res2?.statusText()
|
|
776
|
+
};
|
|
777
|
+
});
|
|
658
778
|
const logInfo = [
|
|
659
779
|
COLORFUL_SYMBOLS.line,
|
|
660
780
|
" ",
|
|
661
781
|
utils.colors.green(`[${statusCode}${statusText ? ` ${statusText}` : ""}]`),
|
|
662
782
|
utils.colors.cyan(url),
|
|
663
|
-
utils.colors.dim(`(in ${duration}
|
|
783
|
+
utils.colors.dim(`(in ${duration}ms)`),
|
|
664
784
|
total.text,
|
|
665
785
|
success.text,
|
|
666
786
|
failed.text,
|
|
@@ -686,23 +806,25 @@ async function checkSiteUrl(url, options) {
|
|
|
686
806
|
if (!info.response && !info.ignored) {
|
|
687
807
|
MEODP.logger.log(COLORFUL_SYMBOLS.line, " ", COLORFUL_SYMBOLS.error, utils.colors.red("Timeout:"), utils.colors.underline(url2));
|
|
688
808
|
MEODP.wLogger.error(` Timeout: ${url2}`);
|
|
809
|
+
siteLinkItem.checkStatus = "timeout";
|
|
689
810
|
}
|
|
690
811
|
}
|
|
691
812
|
let noModuleAssetsPassed = true;
|
|
692
813
|
if (options.checkNoModule) {
|
|
693
814
|
noModuleAssetsPassed = await checkUrlNomoduleAssets(page);
|
|
694
815
|
}
|
|
695
|
-
const urlMapValues = Array.from(urlMap.values());
|
|
696
816
|
const siteUrlItem = siteUrlMap.get(url);
|
|
697
817
|
if (siteUrlItem) {
|
|
698
|
-
|
|
818
|
+
const isAllRequestsPassed = siteLinkItem.requests.filter((value) => !value.ignored).every((value) => value.statusCode && value.statusCode < 400);
|
|
819
|
+
if (isAllRequestsPassed && noModuleAssetsPassed) {
|
|
699
820
|
siteUrlItem.checkStatus = "passed";
|
|
700
|
-
const successCount = Array.from(siteUrlMap.values()).filter((value) => value.checkStatus === "passed").length;
|
|
701
|
-
bar?.update(successCount);
|
|
702
821
|
} else {
|
|
703
822
|
siteUrlItem.checkStatus = "failed";
|
|
704
823
|
}
|
|
824
|
+
siteLinkItem.checkStatus = siteUrlItem.checkStatus;
|
|
705
825
|
}
|
|
826
|
+
siteData.links.push(siteLinkItem);
|
|
827
|
+
await MEODP.db.write();
|
|
706
828
|
if (!isExternalLink) {
|
|
707
829
|
const queue = PQueueMap.get(url);
|
|
708
830
|
if (queue) {
|
|
@@ -723,6 +845,7 @@ async function checkSiteUrl(url, options) {
|
|
|
723
845
|
}
|
|
724
846
|
}
|
|
725
847
|
await page.close();
|
|
848
|
+
return siteLinkItem;
|
|
726
849
|
}
|
|
727
850
|
async function checkSite(props) {
|
|
728
851
|
const { url } = props;
|
|
@@ -739,11 +862,25 @@ async function checkSite(props) {
|
|
|
739
862
|
statusCode: 0,
|
|
740
863
|
checkStatus: "pending"
|
|
741
864
|
});
|
|
865
|
+
await MEODP.db.update((data) => {
|
|
866
|
+
data.sites[homeUrl] = {
|
|
867
|
+
name: props.name,
|
|
868
|
+
url: homeUrl,
|
|
869
|
+
checkStatus: "pending",
|
|
870
|
+
links: [],
|
|
871
|
+
ignored: []
|
|
872
|
+
};
|
|
873
|
+
});
|
|
742
874
|
bar?.setTotal(siteUrlMap.size);
|
|
743
|
-
await checkSiteUrl(homeUrl, {
|
|
875
|
+
const siteLinkItem = await checkSiteUrl(homeUrl, {
|
|
744
876
|
urlItem: props,
|
|
745
877
|
checkNoModule: true
|
|
746
878
|
});
|
|
879
|
+
await MEODP.db.update((data) => {
|
|
880
|
+
const siteItem = data.sites[homeUrl];
|
|
881
|
+
siteItem.title = siteLinkItem?.title;
|
|
882
|
+
siteItem.checkStatus = siteItem.links.every((link) => link.checkStatus === "passed") ? "passed" : "failed";
|
|
883
|
+
});
|
|
747
884
|
await queue.onIdle();
|
|
748
885
|
await page.close();
|
|
749
886
|
await context.close();
|
|
@@ -816,54 +953,6 @@ async function checkMEODPUrl(meodpUrl) {
|
|
|
816
953
|
MEODP.logger.success(`\u{1F577}\uFE0F ${utils.colors.greenBright("[DONE]")} ${utils.colors.gray("in")} ${duration / 1e3}s.`);
|
|
817
954
|
}
|
|
818
955
|
|
|
819
|
-
const defaultMEODPConfig = {
|
|
820
|
-
urls: [],
|
|
821
|
-
concurrency: 10,
|
|
822
|
-
log: {
|
|
823
|
-
type: "raw",
|
|
824
|
-
file: true
|
|
825
|
-
}
|
|
826
|
-
};
|
|
827
|
-
function defineConfig(config) {
|
|
828
|
-
return config;
|
|
829
|
-
}
|
|
830
|
-
async function runByConfig(config) {
|
|
831
|
-
const { urls, debug } = config;
|
|
832
|
-
consola__default.level = debug ? 5 : 3;
|
|
833
|
-
const queue = new PQueue__default({
|
|
834
|
-
concurrency: config.concurrency
|
|
835
|
-
});
|
|
836
|
-
if (config.log?.type === "progress") {
|
|
837
|
-
for (const urlItem of urls) {
|
|
838
|
-
const { type, url, emoji } = getMEODPUrlItemInfo(urlItem);
|
|
839
|
-
const bar = multiBar.create(1, 0, {
|
|
840
|
-
emoji,
|
|
841
|
-
url,
|
|
842
|
-
type,
|
|
843
|
-
error_count: 0
|
|
844
|
-
});
|
|
845
|
-
progressBarMap.set(url, bar);
|
|
846
|
-
}
|
|
847
|
-
const curBar = multiBar.create(1, 0, {
|
|
848
|
-
name: "CURRENT"
|
|
849
|
-
});
|
|
850
|
-
progressBarMap.set("CURRENT", curBar);
|
|
851
|
-
}
|
|
852
|
-
const browser = await getBrowser();
|
|
853
|
-
for (const urlItem of urls) {
|
|
854
|
-
await queue.add(async () => {
|
|
855
|
-
await checkMEODPUrl(urlItem);
|
|
856
|
-
});
|
|
857
|
-
}
|
|
858
|
-
await queue.onIdle();
|
|
859
|
-
if (config.log?.type === "progress") {
|
|
860
|
-
multiBar.stop();
|
|
861
|
-
}
|
|
862
|
-
await browser.close();
|
|
863
|
-
consola__default.debug("browser closed");
|
|
864
|
-
return true;
|
|
865
|
-
}
|
|
866
|
-
|
|
867
956
|
exports.LocalLog = LocalLog;
|
|
868
957
|
exports.MEODP = MEODP;
|
|
869
958
|
exports.PQueueMap = PQueueMap;
|
|
@@ -872,6 +961,7 @@ exports.checkMEODPUrl = checkMEODPUrl;
|
|
|
872
961
|
exports.checkSite = checkSite;
|
|
873
962
|
exports.checkSiteUrl = checkSiteUrl;
|
|
874
963
|
exports.checkUrlNomoduleAssets = checkUrlNomoduleAssets;
|
|
964
|
+
exports.createLowDB = createLowDB;
|
|
875
965
|
exports.createMEODPLogger = createMEODPLogger;
|
|
876
966
|
exports.createWinstonLogger = createWinstonLogger;
|
|
877
967
|
exports.defaultMEODPConfig = defaultMEODPConfig;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "meodp",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.0.
|
|
4
|
+
"version": "0.0.7",
|
|
5
5
|
"packageManager": "pnpm@9.12.3",
|
|
6
6
|
"description": "Mystic Eyes of Death Perception. Find out dead links in your site.",
|
|
7
7
|
"keywords": [
|
|
@@ -29,7 +29,9 @@
|
|
|
29
29
|
"scripts": {
|
|
30
30
|
"build": "unbuild",
|
|
31
31
|
"dev": "unbuild --stub",
|
|
32
|
-
"
|
|
32
|
+
"dev:client": "pnpm -C client run dev",
|
|
33
|
+
"demo": "pnpm -C examples/app run dev",
|
|
34
|
+
"demo:export": "pnpm -C examples/app run export",
|
|
33
35
|
"demo:app": "pnpm -C examples/app run dev",
|
|
34
36
|
"example": "vite-node examples/check-site.ts",
|
|
35
37
|
"test": "npm run test:e2e",
|
|
@@ -50,6 +52,8 @@
|
|
|
50
52
|
"consola": "^3.2.3",
|
|
51
53
|
"date-fns": "^4.1.0",
|
|
52
54
|
"fs-extra": "^11.2.0",
|
|
55
|
+
"lowdb": "^7.0.1",
|
|
56
|
+
"markdown-table": "^3.0.4",
|
|
53
57
|
"p-queue": "^8.0.1",
|
|
54
58
|
"strip-ansi": "^7.1.0",
|
|
55
59
|
"winston": "^3.17.0",
|