meodp 0.0.6 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bin/index.mjs +0 -0
- package/dist/cli/index.cjs +188 -18
- package/dist/cli/index.mjs +174 -7
- package/dist/index.cjs +39 -37
- package/dist/index.d.cts +87 -3
- package/dist/index.d.mts +87 -3
- package/dist/index.d.ts +87 -3
- package/dist/index.mjs +9 -8
- package/dist/shared/{meodp.c2959ba5.cjs → meodp.6b19529e.cjs} +286 -191
- package/dist/shared/{meodp.9c5df07b.mjs → meodp.b58178b8.mjs} +281 -187
- package/package.json +6 -2
|
@@ -1,21 +1,57 @@
|
|
|
1
|
-
import consola from 'consola';
|
|
2
|
-
import PQueue from 'p-queue';
|
|
3
1
|
import { colors } from 'consola/utils';
|
|
4
2
|
import { chromium } from 'playwright';
|
|
5
3
|
import path from 'node:path';
|
|
6
4
|
import process from 'node:process';
|
|
7
|
-
import
|
|
8
|
-
import
|
|
9
|
-
import
|
|
10
|
-
import winston from 'winston';
|
|
5
|
+
import consola from 'consola';
|
|
6
|
+
import { JSONFilePreset } from 'lowdb/node';
|
|
7
|
+
import PQueue from 'p-queue';
|
|
11
8
|
import cliProgress from 'cli-progress';
|
|
12
9
|
import Options from 'cli-progress/lib/options.js';
|
|
13
10
|
import _defaultFormatBar from 'cli-progress/lib/format-bar.js';
|
|
14
11
|
import _defaultFormatTime from 'cli-progress/lib/format-time.js';
|
|
12
|
+
import { formatDate } from 'date-fns';
|
|
13
|
+
import fs from 'fs-extra';
|
|
14
|
+
import stripAnsi from 'strip-ansi';
|
|
15
|
+
import winston from 'winston';
|
|
15
16
|
|
|
16
17
|
const siteUrlMap = /* @__PURE__ */ new Map();
|
|
17
18
|
const PQueueMap = /* @__PURE__ */ new Map();
|
|
18
19
|
|
|
20
|
+
const formatter = (options, params, payload) => {
|
|
21
|
+
const { value, total } = params;
|
|
22
|
+
const formatTime = options.formatTime || _defaultFormatTime;
|
|
23
|
+
const formatBar = options.formatBar || _defaultFormatBar;
|
|
24
|
+
const bar = formatBar(params.progress, options);
|
|
25
|
+
const stopTime = params.stopTime || Date.now();
|
|
26
|
+
const elapsedTime = Math.round((stopTime - params.startTime) / 1e3);
|
|
27
|
+
const duration_formatted = formatTime(elapsedTime, options, 1);
|
|
28
|
+
consola.debug(duration_formatted);
|
|
29
|
+
if (!payload.type) {
|
|
30
|
+
options = Object.assign({}, options, cliProgress.Presets.rect);
|
|
31
|
+
Options.assignDerivedOptions(options);
|
|
32
|
+
const bar2 = formatBar(params.progress, options);
|
|
33
|
+
const { site } = payload;
|
|
34
|
+
const url = payload.url || "";
|
|
35
|
+
return `${bar2} ${value}/${total} | ${colors.yellow(site || "MEODP Item")} | ${colors.cyan(url || "URL")}`;
|
|
36
|
+
} else {
|
|
37
|
+
const { error_count, emoji, url, type } = payload;
|
|
38
|
+
const errorInfo = error_count ? `\u274C${error_count} ` : "";
|
|
39
|
+
return `${bar} \u2705${colors.green(value)}/${total} ${errorInfo}| ${duration_formatted} | ${emoji} ${colors.yellow(type)} | ${colors.cyan(url)}`;
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
const multiBar = new cliProgress.MultiBar({
|
|
43
|
+
hideCursor: true,
|
|
44
|
+
// format: '{bar} {percentage}% | ✅{value}/{total} ❌{error_count} | {duration_formatted} | {name}',
|
|
45
|
+
format: formatter,
|
|
46
|
+
barsize: 20,
|
|
47
|
+
autopadding: true,
|
|
48
|
+
barCompleteChar: "\u2588",
|
|
49
|
+
barIncompleteChar: "\u2591",
|
|
50
|
+
// important! redraw everything to avoid "empty" completed bars
|
|
51
|
+
forceRedraw: true
|
|
52
|
+
}, cliProgress.Presets.rect);
|
|
53
|
+
const progressBarMap = /* @__PURE__ */ new Map();
|
|
54
|
+
|
|
19
55
|
var __defProp$1 = Object.defineProperty;
|
|
20
56
|
var __defNormalProp$1 = (obj, key, value) => key in obj ? __defProp$1(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
21
57
|
var __publicField$1 = (obj, key, value) => {
|
|
@@ -148,40 +184,93 @@ const COLORFUL_SYMBOLS = {
|
|
|
148
184
|
line: colors.dim(RAW_SYMBOLS.line)
|
|
149
185
|
};
|
|
150
186
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
const
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
187
|
+
function createMEODPLogger() {
|
|
188
|
+
return {
|
|
189
|
+
_log: (...args) => {
|
|
190
|
+
const allLogPath = path.resolve(MEODP.logFolder, "all.log");
|
|
191
|
+
LocalLog.log(allLogPath, stripAnsi(args.join(" ")));
|
|
192
|
+
},
|
|
193
|
+
log: (...args) => {
|
|
194
|
+
const content = formatArgs(args);
|
|
195
|
+
consola.log(content);
|
|
196
|
+
MEODP.logger._log(content);
|
|
197
|
+
},
|
|
198
|
+
inner: (...args) => {
|
|
199
|
+
const content = formatArgs(args);
|
|
200
|
+
consola.log(COLORFUL_SYMBOLS.line, " ", content);
|
|
201
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.line, " ", content);
|
|
202
|
+
},
|
|
203
|
+
start: (...args) => {
|
|
204
|
+
const content = formatArgs(args);
|
|
205
|
+
consola.start(content);
|
|
206
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.start, content);
|
|
207
|
+
},
|
|
208
|
+
success: (...args) => {
|
|
209
|
+
const content = formatArgs(args);
|
|
210
|
+
consola.success(content);
|
|
211
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.success, content);
|
|
212
|
+
},
|
|
213
|
+
info: (...args) => {
|
|
214
|
+
const content = formatArgs(args);
|
|
215
|
+
consola.info(content);
|
|
216
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.info, content);
|
|
217
|
+
},
|
|
218
|
+
warn: (...args) => {
|
|
219
|
+
const content = formatArgs(args);
|
|
220
|
+
consola.warn(content);
|
|
221
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.warn, content);
|
|
222
|
+
},
|
|
223
|
+
error: (...args) => {
|
|
224
|
+
const content = formatArgs(args);
|
|
225
|
+
consola.log(COLORFUL_SYMBOLS.error, content);
|
|
226
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.error, content);
|
|
227
|
+
MEODP.wLogger.error(content);
|
|
228
|
+
}
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function createWinstonLogger() {
|
|
233
|
+
const allLogPath = path.resolve(MEODP.logFolder, "all.log");
|
|
234
|
+
const errorLogPath = path.resolve(MEODP.logFolder, "error.log");
|
|
235
|
+
const customFormats = [
|
|
236
|
+
winston.format.timestamp({
|
|
237
|
+
format: "YYYY-MM-DD HH:mm:ss"
|
|
238
|
+
})
|
|
239
|
+
// winston.format.colorize(),
|
|
240
|
+
// winston.format.errors({ stack: true }),
|
|
241
|
+
// winston.format.json(),
|
|
242
|
+
// winston.format.simple(),
|
|
243
|
+
// winston.format.uncolorize(),
|
|
244
|
+
];
|
|
245
|
+
customFormats.push(
|
|
246
|
+
winston.format.printf(({ level, message, timestamp, service }) => {
|
|
247
|
+
const content = [
|
|
248
|
+
timestamp,
|
|
249
|
+
`[${service}]`,
|
|
250
|
+
level.toUpperCase(),
|
|
251
|
+
message
|
|
252
|
+
];
|
|
253
|
+
return content.join(" ");
|
|
254
|
+
})
|
|
255
|
+
);
|
|
256
|
+
const customTransports = [
|
|
257
|
+
new winston.transports.File({
|
|
258
|
+
filename: errorLogPath,
|
|
259
|
+
level: "error"
|
|
260
|
+
}),
|
|
261
|
+
new winston.transports.File({
|
|
262
|
+
filename: allLogPath
|
|
263
|
+
// level: 'info',
|
|
264
|
+
})
|
|
265
|
+
];
|
|
266
|
+
return winston.createLogger({
|
|
267
|
+
level: "info",
|
|
268
|
+
format: winston.format.combine(...customFormats),
|
|
269
|
+
defaultMeta: { service: "MEODP" },
|
|
270
|
+
transports: customTransports,
|
|
271
|
+
exitOnError: false
|
|
272
|
+
});
|
|
273
|
+
}
|
|
185
274
|
|
|
186
275
|
function formatArgs(args) {
|
|
187
276
|
return args.join(" ");
|
|
@@ -192,7 +281,7 @@ const TIME_COLOR = {
|
|
|
192
281
|
slow: colors.red
|
|
193
282
|
};
|
|
194
283
|
function getFormattedDuration(duration) {
|
|
195
|
-
const durationText = `${(
|
|
284
|
+
const durationText = `${(duration / 1e3).toString()}s`;
|
|
196
285
|
let type;
|
|
197
286
|
if (duration < 500) {
|
|
198
287
|
type = "fast";
|
|
@@ -209,9 +298,12 @@ function getFormattedDataFromResponse(res) {
|
|
|
209
298
|
const statusInfo = statusText ? `${statusCode?.toString()} ${statusText}` : statusCode?.toString();
|
|
210
299
|
const statusInfoText = statusCode >= 400 ? colors.red(`[${statusInfo}]`) : colors.green(`[${statusInfo}]`);
|
|
211
300
|
const req = res.request();
|
|
212
|
-
const duration = req.timing().responseEnd - req.timing().requestStart;
|
|
301
|
+
const duration = Math.round(req.timing().responseEnd - req.timing().requestStart);
|
|
213
302
|
const durationText = `${COLORFUL_SYMBOLS.line} ${getFormattedDuration(duration)} ${COLORFUL_SYMBOLS.line}`;
|
|
214
303
|
return {
|
|
304
|
+
/**
|
|
305
|
+
* 四舍五入 ms
|
|
306
|
+
*/
|
|
215
307
|
duration,
|
|
216
308
|
durationText,
|
|
217
309
|
statusCode,
|
|
@@ -359,92 +451,62 @@ function getMEODPUrlItemInfo(urlItem) {
|
|
|
359
451
|
};
|
|
360
452
|
}
|
|
361
453
|
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
const content = formatArgs(args);
|
|
390
|
-
consola.info(content);
|
|
391
|
-
MEODP.logger._log(COLORFUL_SYMBOLS.info, content);
|
|
392
|
-
},
|
|
393
|
-
warn: (...args) => {
|
|
394
|
-
const content = formatArgs(args);
|
|
395
|
-
consola.warn(content);
|
|
396
|
-
MEODP.logger._log(COLORFUL_SYMBOLS.warn, content);
|
|
397
|
-
},
|
|
398
|
-
error: (...args) => {
|
|
399
|
-
const content = formatArgs(args);
|
|
400
|
-
consola.log(COLORFUL_SYMBOLS.error, content);
|
|
401
|
-
MEODP.logger._log(COLORFUL_SYMBOLS.error, content);
|
|
402
|
-
MEODP.wLogger.error(content);
|
|
454
|
+
const defaultMEODPConfig = {
|
|
455
|
+
urls: [],
|
|
456
|
+
concurrency: 10,
|
|
457
|
+
log: {
|
|
458
|
+
type: "raw",
|
|
459
|
+
file: true
|
|
460
|
+
}
|
|
461
|
+
};
|
|
462
|
+
function defineConfig(config) {
|
|
463
|
+
return config;
|
|
464
|
+
}
|
|
465
|
+
async function runByConfig(config) {
|
|
466
|
+
const { urls, debug } = config;
|
|
467
|
+
consola.level = debug ? 5 : 3;
|
|
468
|
+
const queue = new PQueue({
|
|
469
|
+
concurrency: config.concurrency
|
|
470
|
+
});
|
|
471
|
+
if (config.log?.type === "progress") {
|
|
472
|
+
for (const urlItem of urls) {
|
|
473
|
+
const { type, url, emoji } = getMEODPUrlItemInfo(urlItem);
|
|
474
|
+
const bar = multiBar.create(1, 0, {
|
|
475
|
+
emoji,
|
|
476
|
+
url,
|
|
477
|
+
type,
|
|
478
|
+
error_count: 0
|
|
479
|
+
});
|
|
480
|
+
progressBarMap.set(url, bar);
|
|
403
481
|
}
|
|
404
|
-
|
|
482
|
+
const curBar = multiBar.create(1, 0, {
|
|
483
|
+
name: "CURRENT"
|
|
484
|
+
});
|
|
485
|
+
progressBarMap.set("CURRENT", curBar);
|
|
486
|
+
}
|
|
487
|
+
const browser = await getBrowser();
|
|
488
|
+
for (const urlItem of urls) {
|
|
489
|
+
await queue.add(async () => {
|
|
490
|
+
await checkMEODPUrl(urlItem);
|
|
491
|
+
});
|
|
492
|
+
}
|
|
493
|
+
await queue.onIdle();
|
|
494
|
+
if (config.log?.type === "progress") {
|
|
495
|
+
multiBar.stop();
|
|
496
|
+
}
|
|
497
|
+
await browser.close();
|
|
498
|
+
consola.debug("browser closed");
|
|
499
|
+
return true;
|
|
405
500
|
}
|
|
406
501
|
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
// winston.format.errors({ stack: true }),
|
|
416
|
-
// winston.format.json(),
|
|
417
|
-
// winston.format.simple(),
|
|
418
|
-
// winston.format.uncolorize(),
|
|
419
|
-
];
|
|
420
|
-
customFormats.push(
|
|
421
|
-
winston.format.printf(({ level, message, timestamp, service }) => {
|
|
422
|
-
const content = [
|
|
423
|
-
timestamp,
|
|
424
|
-
`[${service}]`,
|
|
425
|
-
level.toUpperCase(),
|
|
426
|
-
message
|
|
427
|
-
];
|
|
428
|
-
return content.join(" ");
|
|
429
|
-
})
|
|
430
|
-
);
|
|
431
|
-
const customTransports = [
|
|
432
|
-
new winston.transports.File({
|
|
433
|
-
filename: errorLogPath,
|
|
434
|
-
level: "error"
|
|
435
|
-
}),
|
|
436
|
-
new winston.transports.File({
|
|
437
|
-
filename: allLogPath
|
|
438
|
-
// level: 'info',
|
|
439
|
-
})
|
|
440
|
-
];
|
|
441
|
-
return winston.createLogger({
|
|
442
|
-
level: "info",
|
|
443
|
-
format: winston.format.combine(...customFormats),
|
|
444
|
-
defaultMeta: { service: "MEODP" },
|
|
445
|
-
transports: customTransports,
|
|
446
|
-
exitOnError: false
|
|
447
|
-
});
|
|
502
|
+
const defaultData = {
|
|
503
|
+
config: defaultMEODPConfig,
|
|
504
|
+
sites: {}
|
|
505
|
+
};
|
|
506
|
+
async function createLowDB(rootDir = process.cwd()) {
|
|
507
|
+
const dbJsonPath = path.resolve(rootDir, "logs/meodp/db.json");
|
|
508
|
+
const db = await JSONFilePreset(dbJsonPath, defaultData);
|
|
509
|
+
return db;
|
|
448
510
|
}
|
|
449
511
|
|
|
450
512
|
var __defProp = Object.defineProperty;
|
|
@@ -469,6 +531,9 @@ const _MEODP = class _MEODP {
|
|
|
469
531
|
await LocalLog.init();
|
|
470
532
|
_MEODP.logFolder = LocalLog.logFolder || path.resolve(process.cwd(), "logs/meodp");
|
|
471
533
|
_MEODP.wLogger = createWinstonLogger();
|
|
534
|
+
const db = await createLowDB();
|
|
535
|
+
_MEODP.db = db;
|
|
536
|
+
await db.update((data) => data.config = _MEODP.config);
|
|
472
537
|
}
|
|
473
538
|
/**
|
|
474
539
|
* filter link by ignoreLinks
|
|
@@ -517,6 +582,7 @@ __publicField(_MEODP, "browser");
|
|
|
517
582
|
__publicField(_MEODP, "logFolder");
|
|
518
583
|
__publicField(_MEODP, "logger", createMEODPLogger());
|
|
519
584
|
__publicField(_MEODP, "wLogger");
|
|
585
|
+
__publicField(_MEODP, "db");
|
|
520
586
|
let MEODP = _MEODP;
|
|
521
587
|
|
|
522
588
|
async function getBrowser() {
|
|
@@ -565,7 +631,6 @@ async function checkUrlNomoduleAssets(page) {
|
|
|
565
631
|
if (!nomoduleUrl) {
|
|
566
632
|
return;
|
|
567
633
|
}
|
|
568
|
-
const startTime = Date.now();
|
|
569
634
|
const newPage = await context.newPage();
|
|
570
635
|
const response = await newPage.goto(nomoduleUrl);
|
|
571
636
|
if (!response) {
|
|
@@ -576,7 +641,8 @@ async function checkUrlNomoduleAssets(page) {
|
|
|
576
641
|
statusInfo,
|
|
577
642
|
linkText
|
|
578
643
|
} = getFormattedDataFromResponse(response);
|
|
579
|
-
const
|
|
644
|
+
const req = response.request();
|
|
645
|
+
const duration = req.timing().responseEnd - req.timing().requestStart;
|
|
580
646
|
const durationTxt = colors.dim(`\u2502${colors.blue(`${duration.toString().padStart(4, " ")}${colors.white("ms")}`)} \u2502`);
|
|
581
647
|
if (statusCode >= 400) {
|
|
582
648
|
MEODP.logger.inner(" ", COLORFUL_SYMBOLS.error, colors.red(statusInfo), durationTxt, linkText);
|
|
@@ -594,9 +660,31 @@ async function checkSiteUrl(url, options) {
|
|
|
594
660
|
const browser = await getBrowser();
|
|
595
661
|
const page = await browser.newPage();
|
|
596
662
|
const bar = progressBarMap.get(options.urlItem.url);
|
|
597
|
-
const startTime = Date.now();
|
|
598
663
|
const isExternalLink = MEODP.isExternalLink(url, options.urlItem);
|
|
599
664
|
const log = LocalLog.createLog(options.urlItem);
|
|
665
|
+
const siteData = MEODP.db.data.sites[options.urlItem.url];
|
|
666
|
+
const siteLinkItem = {
|
|
667
|
+
url,
|
|
668
|
+
statusCode: 0,
|
|
669
|
+
checkStatus: "pending",
|
|
670
|
+
success: 0,
|
|
671
|
+
failed: 0,
|
|
672
|
+
ignored: 0,
|
|
673
|
+
timeout: 0,
|
|
674
|
+
duration: 0,
|
|
675
|
+
total: 0,
|
|
676
|
+
requests: []
|
|
677
|
+
};
|
|
678
|
+
if (MEODP.isIgnoredLink(url)) {
|
|
679
|
+
siteUrlMap.set(url, {
|
|
680
|
+
statusCode: 0,
|
|
681
|
+
checkStatus: "ignored"
|
|
682
|
+
});
|
|
683
|
+
MEODP.logger.inner(`${colors.dim(`[Ignored] ${colors.underline(url)}`)}`);
|
|
684
|
+
siteData.ignored.push(url);
|
|
685
|
+
await MEODP.db.write();
|
|
686
|
+
return;
|
|
687
|
+
}
|
|
600
688
|
if (!MEODP.config.checkExternalLinks && isExternalLink) {
|
|
601
689
|
let res = null;
|
|
602
690
|
try {
|
|
@@ -604,13 +692,19 @@ async function checkSiteUrl(url, options) {
|
|
|
604
692
|
waitUntil: "load"
|
|
605
693
|
});
|
|
606
694
|
} catch (e) {
|
|
607
|
-
|
|
695
|
+
siteLinkItem.timeout = 1;
|
|
696
|
+
MEODP.logger.inner(
|
|
697
|
+
colors.red("[Timeout]"),
|
|
698
|
+
colors.underline(url)
|
|
699
|
+
);
|
|
700
|
+
consola.debug(e);
|
|
608
701
|
}
|
|
609
702
|
if (!res)
|
|
610
703
|
return;
|
|
611
704
|
const {
|
|
612
705
|
statusCode,
|
|
613
706
|
statusInfoText,
|
|
707
|
+
duration,
|
|
614
708
|
durationText
|
|
615
709
|
} = getFormattedDataFromResponse(res);
|
|
616
710
|
const checkStatus = statusCode < 400 ? "passed" : "failed";
|
|
@@ -623,28 +717,59 @@ async function checkSiteUrl(url, options) {
|
|
|
623
717
|
if (statusCode >= 400) {
|
|
624
718
|
MEODP.wLogger.error(` ${statusInfoText} ${url} ${durationText}`);
|
|
625
719
|
}
|
|
720
|
+
siteLinkItem.statusCode = statusCode;
|
|
721
|
+
siteLinkItem.checkStatus = checkStatus;
|
|
722
|
+
siteLinkItem.duration = duration;
|
|
723
|
+
siteLinkItem.total = 1;
|
|
724
|
+
siteLinkItem.success = checkStatus === "passed" ? 1 : 0;
|
|
725
|
+
siteLinkItem.failed = checkStatus === "failed" ? 1 : 0;
|
|
726
|
+
siteLinkItem.requests = [];
|
|
727
|
+
siteData.links.push(siteLinkItem);
|
|
728
|
+
await MEODP.db.write();
|
|
626
729
|
return;
|
|
627
730
|
}
|
|
628
731
|
const urlMap = registerPageEvents(page, options.urlItem);
|
|
629
|
-
const { success, failed, total, ignored, timeout } = parseUrlMap(urlMap);
|
|
630
732
|
try {
|
|
631
733
|
const res = await page.goto(url, {
|
|
632
734
|
waitUntil: "networkidle"
|
|
633
735
|
});
|
|
634
736
|
const statusCode = res?.status() || 0;
|
|
737
|
+
const statusText = res?.statusText();
|
|
738
|
+
const req = res?.request();
|
|
739
|
+
const duration = Math.round((req?.timing()?.responseEnd || 0) - (req?.timing()?.requestStart || 0));
|
|
635
740
|
siteUrlMap.set(url, {
|
|
636
741
|
response: res,
|
|
637
742
|
statusCode,
|
|
638
743
|
checkStatus: "goto"
|
|
639
744
|
});
|
|
640
|
-
const
|
|
641
|
-
|
|
745
|
+
const { success, failed, total, ignored, timeout } = parseUrlMap(urlMap);
|
|
746
|
+
siteLinkItem.title = await page.title();
|
|
747
|
+
siteLinkItem.statusCode = statusCode;
|
|
748
|
+
siteLinkItem.statusText = statusText;
|
|
749
|
+
siteLinkItem.checkStatus = "goto";
|
|
750
|
+
siteLinkItem.duration = duration;
|
|
751
|
+
siteLinkItem.total = total.count;
|
|
752
|
+
siteLinkItem.success = success.count;
|
|
753
|
+
siteLinkItem.failed = failed.count;
|
|
754
|
+
siteLinkItem.timeout = timeout.count;
|
|
755
|
+
siteLinkItem.ignored = ignored.count;
|
|
756
|
+
siteLinkItem.requests = Array.from(urlMap.values()).map((value) => {
|
|
757
|
+
const res2 = value.response;
|
|
758
|
+
return {
|
|
759
|
+
failed: value.failed,
|
|
760
|
+
ignored: value.ignored,
|
|
761
|
+
// res.url 还没用返回
|
|
762
|
+
url: req?.url(),
|
|
763
|
+
statusCode: res2?.status(),
|
|
764
|
+
statusText: res2?.statusText()
|
|
765
|
+
};
|
|
766
|
+
});
|
|
642
767
|
const logInfo = [
|
|
643
768
|
COLORFUL_SYMBOLS.line,
|
|
644
769
|
" ",
|
|
645
770
|
colors.green(`[${statusCode}${statusText ? ` ${statusText}` : ""}]`),
|
|
646
771
|
colors.cyan(url),
|
|
647
|
-
colors.dim(`(in ${duration}
|
|
772
|
+
colors.dim(`(in ${duration}ms)`),
|
|
648
773
|
total.text,
|
|
649
774
|
success.text,
|
|
650
775
|
failed.text,
|
|
@@ -670,23 +795,25 @@ async function checkSiteUrl(url, options) {
|
|
|
670
795
|
if (!info.response && !info.ignored) {
|
|
671
796
|
MEODP.logger.log(COLORFUL_SYMBOLS.line, " ", COLORFUL_SYMBOLS.error, colors.red("Timeout:"), colors.underline(url2));
|
|
672
797
|
MEODP.wLogger.error(` Timeout: ${url2}`);
|
|
798
|
+
siteLinkItem.checkStatus = "timeout";
|
|
673
799
|
}
|
|
674
800
|
}
|
|
675
801
|
let noModuleAssetsPassed = true;
|
|
676
802
|
if (options.checkNoModule) {
|
|
677
803
|
noModuleAssetsPassed = await checkUrlNomoduleAssets(page);
|
|
678
804
|
}
|
|
679
|
-
const urlMapValues = Array.from(urlMap.values());
|
|
680
805
|
const siteUrlItem = siteUrlMap.get(url);
|
|
681
806
|
if (siteUrlItem) {
|
|
682
|
-
|
|
807
|
+
const isAllRequestsPassed = siteLinkItem.requests.filter((value) => !value.ignored).every((value) => value.statusCode && value.statusCode < 400);
|
|
808
|
+
if (isAllRequestsPassed && noModuleAssetsPassed) {
|
|
683
809
|
siteUrlItem.checkStatus = "passed";
|
|
684
|
-
const successCount = Array.from(siteUrlMap.values()).filter((value) => value.checkStatus === "passed").length;
|
|
685
|
-
bar?.update(successCount);
|
|
686
810
|
} else {
|
|
687
811
|
siteUrlItem.checkStatus = "failed";
|
|
688
812
|
}
|
|
813
|
+
siteLinkItem.checkStatus = siteUrlItem.checkStatus;
|
|
689
814
|
}
|
|
815
|
+
siteData.links.push(siteLinkItem);
|
|
816
|
+
await MEODP.db.write();
|
|
690
817
|
if (!isExternalLink) {
|
|
691
818
|
const queue = PQueueMap.get(url);
|
|
692
819
|
if (queue) {
|
|
@@ -707,6 +834,7 @@ async function checkSiteUrl(url, options) {
|
|
|
707
834
|
}
|
|
708
835
|
}
|
|
709
836
|
await page.close();
|
|
837
|
+
return siteLinkItem;
|
|
710
838
|
}
|
|
711
839
|
async function checkSite(props) {
|
|
712
840
|
const { url } = props;
|
|
@@ -723,11 +851,25 @@ async function checkSite(props) {
|
|
|
723
851
|
statusCode: 0,
|
|
724
852
|
checkStatus: "pending"
|
|
725
853
|
});
|
|
854
|
+
await MEODP.db.update((data) => {
|
|
855
|
+
data.sites[homeUrl] = {
|
|
856
|
+
name: props.name,
|
|
857
|
+
url: homeUrl,
|
|
858
|
+
checkStatus: "pending",
|
|
859
|
+
links: [],
|
|
860
|
+
ignored: []
|
|
861
|
+
};
|
|
862
|
+
});
|
|
726
863
|
bar?.setTotal(siteUrlMap.size);
|
|
727
|
-
await checkSiteUrl(homeUrl, {
|
|
864
|
+
const siteLinkItem = await checkSiteUrl(homeUrl, {
|
|
728
865
|
urlItem: props,
|
|
729
866
|
checkNoModule: true
|
|
730
867
|
});
|
|
868
|
+
await MEODP.db.update((data) => {
|
|
869
|
+
const siteItem = data.sites[homeUrl];
|
|
870
|
+
siteItem.title = siteLinkItem?.title;
|
|
871
|
+
siteItem.checkStatus = siteItem.links.every((link) => link.checkStatus === "passed") ? "passed" : "failed";
|
|
872
|
+
});
|
|
731
873
|
await queue.onIdle();
|
|
732
874
|
await page.close();
|
|
733
875
|
await context.close();
|
|
@@ -800,52 +942,4 @@ async function checkMEODPUrl(meodpUrl) {
|
|
|
800
942
|
MEODP.logger.success(`\u{1F577}\uFE0F ${colors.greenBright("[DONE]")} ${colors.gray("in")} ${duration / 1e3}s.`);
|
|
801
943
|
}
|
|
802
944
|
|
|
803
|
-
|
|
804
|
-
urls: [],
|
|
805
|
-
concurrency: 10,
|
|
806
|
-
log: {
|
|
807
|
-
type: "raw",
|
|
808
|
-
file: true
|
|
809
|
-
}
|
|
810
|
-
};
|
|
811
|
-
function defineConfig(config) {
|
|
812
|
-
return config;
|
|
813
|
-
}
|
|
814
|
-
async function runByConfig(config) {
|
|
815
|
-
const { urls, debug } = config;
|
|
816
|
-
consola.level = debug ? 5 : 3;
|
|
817
|
-
const queue = new PQueue({
|
|
818
|
-
concurrency: config.concurrency
|
|
819
|
-
});
|
|
820
|
-
if (config.log?.type === "progress") {
|
|
821
|
-
for (const urlItem of urls) {
|
|
822
|
-
const { type, url, emoji } = getMEODPUrlItemInfo(urlItem);
|
|
823
|
-
const bar = multiBar.create(1, 0, {
|
|
824
|
-
emoji,
|
|
825
|
-
url,
|
|
826
|
-
type,
|
|
827
|
-
error_count: 0
|
|
828
|
-
});
|
|
829
|
-
progressBarMap.set(url, bar);
|
|
830
|
-
}
|
|
831
|
-
const curBar = multiBar.create(1, 0, {
|
|
832
|
-
name: "CURRENT"
|
|
833
|
-
});
|
|
834
|
-
progressBarMap.set("CURRENT", curBar);
|
|
835
|
-
}
|
|
836
|
-
const browser = await getBrowser();
|
|
837
|
-
for (const urlItem of urls) {
|
|
838
|
-
await queue.add(async () => {
|
|
839
|
-
await checkMEODPUrl(urlItem);
|
|
840
|
-
});
|
|
841
|
-
}
|
|
842
|
-
await queue.onIdle();
|
|
843
|
-
if (config.log?.type === "progress") {
|
|
844
|
-
multiBar.stop();
|
|
845
|
-
}
|
|
846
|
-
await browser.close();
|
|
847
|
-
consola.debug("browser closed");
|
|
848
|
-
return true;
|
|
849
|
-
}
|
|
850
|
-
|
|
851
|
-
export { LocalLog as L, MEODP as M, PQueueMap as P, checkLink as a, getSiteLinks as b, checkMEODPUrl as c, checkUrlNomoduleAssets as d, checkSiteUrl as e, checkSite as f, getBrowser as g, defaultMEODPConfig as h, defineConfig as i, localLog as j, createMEODPLogger as k, logUrlItemInfo as l, createWinstonLogger as m, multiBar as n, registerPageEvents as o, progressBarMap as p, formatArgs as q, runByConfig as r, siteUrlMap as s, getFormattedDuration as t, getFormattedDataFromResponse as u, isLink as v, parseUrlMap as w, emojiMap as x, getMEODPUrlItemInfo as y };
|
|
945
|
+
export { LocalLog as L, MEODP as M, PQueueMap as P, checkLink as a, getSiteLinks as b, checkMEODPUrl as c, checkUrlNomoduleAssets as d, checkSiteUrl as e, checkSite as f, getBrowser as g, defaultMEODPConfig as h, defineConfig as i, createLowDB as j, localLog as k, logUrlItemInfo as l, createMEODPLogger as m, createWinstonLogger as n, multiBar as o, progressBarMap as p, registerPageEvents as q, runByConfig as r, siteUrlMap as s, formatArgs as t, getFormattedDuration as u, getFormattedDataFromResponse as v, isLink as w, parseUrlMap as x, emojiMap as y, getMEODPUrlItemInfo as z };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "meodp",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.0.
|
|
4
|
+
"version": "0.0.8",
|
|
5
5
|
"packageManager": "pnpm@9.12.3",
|
|
6
6
|
"description": "Mystic Eyes of Death Perception. Find out dead links in your site.",
|
|
7
7
|
"keywords": [
|
|
@@ -29,7 +29,9 @@
|
|
|
29
29
|
"scripts": {
|
|
30
30
|
"build": "unbuild",
|
|
31
31
|
"dev": "unbuild --stub",
|
|
32
|
-
"
|
|
32
|
+
"dev:client": "pnpm -C client run dev",
|
|
33
|
+
"demo": "pnpm -C examples/app run dev",
|
|
34
|
+
"demo:export": "pnpm -C examples/app run export",
|
|
33
35
|
"demo:app": "pnpm -C examples/app run dev",
|
|
34
36
|
"example": "vite-node examples/check-site.ts",
|
|
35
37
|
"test": "npm run test:e2e",
|
|
@@ -50,6 +52,8 @@
|
|
|
50
52
|
"consola": "^3.2.3",
|
|
51
53
|
"date-fns": "^4.1.0",
|
|
52
54
|
"fs-extra": "^11.2.0",
|
|
55
|
+
"lowdb": "^7.0.1",
|
|
56
|
+
"markdown-table": "^3.0.4",
|
|
53
57
|
"p-queue": "^8.0.1",
|
|
54
58
|
"strip-ansi": "^7.1.0",
|
|
55
59
|
"winston": "^3.17.0",
|