meodp 0.0.5 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/bin/index.mjs +0 -0
- package/dist/cli/index.cjs +168 -18
- package/dist/cli/index.mjs +154 -7
- package/dist/index.cjs +39 -36
- package/dist/index.d.cts +89 -3
- package/dist/index.d.mts +89 -3
- package/dist/index.d.ts +89 -3
- package/dist/index.mjs +9 -8
- package/dist/shared/{meodp.f86a1a9f.mjs → meodp.5489574b.mjs} +313 -191
- package/dist/shared/{meodp.e8cfff87.cjs → meodp.9bf8b8ff.cjs} +319 -195
- package/package.json +6 -2
|
@@ -1,21 +1,57 @@
|
|
|
1
|
-
import consola from 'consola';
|
|
2
|
-
import PQueue from 'p-queue';
|
|
3
1
|
import { colors } from 'consola/utils';
|
|
4
2
|
import { chromium } from 'playwright';
|
|
5
3
|
import path from 'node:path';
|
|
6
4
|
import process from 'node:process';
|
|
7
|
-
import
|
|
8
|
-
import
|
|
9
|
-
import
|
|
10
|
-
import winston from 'winston';
|
|
5
|
+
import consola from 'consola';
|
|
6
|
+
import { JSONFilePreset } from 'lowdb/node';
|
|
7
|
+
import PQueue from 'p-queue';
|
|
11
8
|
import cliProgress from 'cli-progress';
|
|
12
9
|
import Options from 'cli-progress/lib/options.js';
|
|
13
10
|
import _defaultFormatBar from 'cli-progress/lib/format-bar.js';
|
|
14
11
|
import _defaultFormatTime from 'cli-progress/lib/format-time.js';
|
|
12
|
+
import { formatDate } from 'date-fns';
|
|
13
|
+
import fs from 'fs-extra';
|
|
14
|
+
import stripAnsi from 'strip-ansi';
|
|
15
|
+
import winston from 'winston';
|
|
15
16
|
|
|
16
17
|
const siteUrlMap = /* @__PURE__ */ new Map();
|
|
17
18
|
const PQueueMap = /* @__PURE__ */ new Map();
|
|
18
19
|
|
|
20
|
+
const formatter = (options, params, payload) => {
|
|
21
|
+
const { value, total } = params;
|
|
22
|
+
const formatTime = options.formatTime || _defaultFormatTime;
|
|
23
|
+
const formatBar = options.formatBar || _defaultFormatBar;
|
|
24
|
+
const bar = formatBar(params.progress, options);
|
|
25
|
+
const stopTime = params.stopTime || Date.now();
|
|
26
|
+
const elapsedTime = Math.round((stopTime - params.startTime) / 1e3);
|
|
27
|
+
const duration_formatted = formatTime(elapsedTime, options, 1);
|
|
28
|
+
consola.debug(duration_formatted);
|
|
29
|
+
if (!payload.type) {
|
|
30
|
+
options = Object.assign({}, options, cliProgress.Presets.rect);
|
|
31
|
+
Options.assignDerivedOptions(options);
|
|
32
|
+
const bar2 = formatBar(params.progress, options);
|
|
33
|
+
const { site } = payload;
|
|
34
|
+
const url = payload.url || "";
|
|
35
|
+
return `${bar2} ${value}/${total} | ${colors.yellow(site || "MEODP Item")} | ${colors.cyan(url || "URL")}`;
|
|
36
|
+
} else {
|
|
37
|
+
const { error_count, emoji, url, type } = payload;
|
|
38
|
+
const errorInfo = error_count ? `\u274C${error_count} ` : "";
|
|
39
|
+
return `${bar} \u2705${colors.green(value)}/${total} ${errorInfo}| ${duration_formatted} | ${emoji} ${colors.yellow(type)} | ${colors.cyan(url)}`;
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
const multiBar = new cliProgress.MultiBar({
|
|
43
|
+
hideCursor: true,
|
|
44
|
+
// format: '{bar} {percentage}% | ✅{value}/{total} ❌{error_count} | {duration_formatted} | {name}',
|
|
45
|
+
format: formatter,
|
|
46
|
+
barsize: 20,
|
|
47
|
+
autopadding: true,
|
|
48
|
+
barCompleteChar: "\u2588",
|
|
49
|
+
barIncompleteChar: "\u2591",
|
|
50
|
+
// important! redraw everything to avoid "empty" completed bars
|
|
51
|
+
forceRedraw: true
|
|
52
|
+
}, cliProgress.Presets.rect);
|
|
53
|
+
const progressBarMap = /* @__PURE__ */ new Map();
|
|
54
|
+
|
|
19
55
|
var __defProp$1 = Object.defineProperty;
|
|
20
56
|
var __defNormalProp$1 = (obj, key, value) => key in obj ? __defProp$1(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
21
57
|
var __publicField$1 = (obj, key, value) => {
|
|
@@ -65,7 +101,11 @@ const _LocalLog = class _LocalLog {
|
|
|
65
101
|
* @desc append log
|
|
66
102
|
*/
|
|
67
103
|
static async SiteLog(urlItem, ...args) {
|
|
68
|
-
|
|
104
|
+
let id = urlItem.name || urlItem.url;
|
|
105
|
+
if (id.endsWith("/")) {
|
|
106
|
+
id = id.slice(0, -1);
|
|
107
|
+
}
|
|
108
|
+
const logName = id.replace("https://", "").replace("http://", "").replace(/\//g, "-");
|
|
69
109
|
const logPath = path.resolve(this.logFolder, urlItem.type, `${logName}.log`);
|
|
70
110
|
await _LocalLog.log(logPath, ...args);
|
|
71
111
|
}
|
|
@@ -144,57 +184,132 @@ const COLORFUL_SYMBOLS = {
|
|
|
144
184
|
line: colors.dim(RAW_SYMBOLS.line)
|
|
145
185
|
};
|
|
146
186
|
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
const
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
187
|
+
function createMEODPLogger() {
|
|
188
|
+
return {
|
|
189
|
+
_log: (...args) => {
|
|
190
|
+
const allLogPath = path.resolve(MEODP.logFolder, "all.log");
|
|
191
|
+
LocalLog.log(allLogPath, stripAnsi(args.join(" ")));
|
|
192
|
+
},
|
|
193
|
+
log: (...args) => {
|
|
194
|
+
const content = formatArgs(args);
|
|
195
|
+
consola.log(content);
|
|
196
|
+
MEODP.logger._log(content);
|
|
197
|
+
},
|
|
198
|
+
inner: (...args) => {
|
|
199
|
+
const content = formatArgs(args);
|
|
200
|
+
consola.log(COLORFUL_SYMBOLS.line, " ", content);
|
|
201
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.line, " ", content);
|
|
202
|
+
},
|
|
203
|
+
start: (...args) => {
|
|
204
|
+
const content = formatArgs(args);
|
|
205
|
+
consola.start(content);
|
|
206
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.start, content);
|
|
207
|
+
},
|
|
208
|
+
success: (...args) => {
|
|
209
|
+
const content = formatArgs(args);
|
|
210
|
+
consola.success(content);
|
|
211
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.success, content);
|
|
212
|
+
},
|
|
213
|
+
info: (...args) => {
|
|
214
|
+
const content = formatArgs(args);
|
|
215
|
+
consola.info(content);
|
|
216
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.info, content);
|
|
217
|
+
},
|
|
218
|
+
warn: (...args) => {
|
|
219
|
+
const content = formatArgs(args);
|
|
220
|
+
consola.warn(content);
|
|
221
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.warn, content);
|
|
222
|
+
},
|
|
223
|
+
error: (...args) => {
|
|
224
|
+
const content = formatArgs(args);
|
|
225
|
+
consola.log(COLORFUL_SYMBOLS.error, content);
|
|
226
|
+
MEODP.logger._log(COLORFUL_SYMBOLS.error, content);
|
|
227
|
+
MEODP.wLogger.error(content);
|
|
228
|
+
}
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function createWinstonLogger() {
|
|
233
|
+
const allLogPath = path.resolve(MEODP.logFolder, "all.log");
|
|
234
|
+
const errorLogPath = path.resolve(MEODP.logFolder, "error.log");
|
|
235
|
+
const customFormats = [
|
|
236
|
+
winston.format.timestamp({
|
|
237
|
+
format: "YYYY-MM-DD HH:mm:ss"
|
|
238
|
+
})
|
|
239
|
+
// winston.format.colorize(),
|
|
240
|
+
// winston.format.errors({ stack: true }),
|
|
241
|
+
// winston.format.json(),
|
|
242
|
+
// winston.format.simple(),
|
|
243
|
+
// winston.format.uncolorize(),
|
|
244
|
+
];
|
|
245
|
+
customFormats.push(
|
|
246
|
+
winston.format.printf(({ level, message, timestamp, service }) => {
|
|
247
|
+
const content = [
|
|
248
|
+
timestamp,
|
|
249
|
+
`[${service}]`,
|
|
250
|
+
level.toUpperCase(),
|
|
251
|
+
message
|
|
252
|
+
];
|
|
253
|
+
return content.join(" ");
|
|
254
|
+
})
|
|
255
|
+
);
|
|
256
|
+
const customTransports = [
|
|
257
|
+
new winston.transports.File({
|
|
258
|
+
filename: errorLogPath,
|
|
259
|
+
level: "error"
|
|
260
|
+
}),
|
|
261
|
+
new winston.transports.File({
|
|
262
|
+
filename: allLogPath
|
|
263
|
+
// level: 'info',
|
|
264
|
+
})
|
|
265
|
+
];
|
|
266
|
+
return winston.createLogger({
|
|
267
|
+
level: "info",
|
|
268
|
+
format: winston.format.combine(...customFormats),
|
|
269
|
+
defaultMeta: { service: "MEODP" },
|
|
270
|
+
transports: customTransports,
|
|
271
|
+
exitOnError: false
|
|
272
|
+
});
|
|
273
|
+
}
|
|
181
274
|
|
|
182
275
|
function formatArgs(args) {
|
|
183
276
|
return args.join(" ");
|
|
184
277
|
}
|
|
278
|
+
const TIME_COLOR = {
|
|
279
|
+
fast: colors.green,
|
|
280
|
+
medium: colors.yellow,
|
|
281
|
+
slow: colors.red
|
|
282
|
+
};
|
|
283
|
+
function getFormattedDuration(duration) {
|
|
284
|
+
const durationText = `${(duration / 1e3).toString()}s`;
|
|
285
|
+
let type;
|
|
286
|
+
if (duration < 500) {
|
|
287
|
+
type = "fast";
|
|
288
|
+
} else if (duration < 3e3) {
|
|
289
|
+
type = "medium";
|
|
290
|
+
} else {
|
|
291
|
+
type = "slow";
|
|
292
|
+
}
|
|
293
|
+
return TIME_COLOR[type](durationText);
|
|
294
|
+
}
|
|
185
295
|
function getFormattedDataFromResponse(res) {
|
|
186
296
|
const statusCode = res?.status();
|
|
187
297
|
const statusText = res?.statusText();
|
|
188
298
|
const statusInfo = statusText ? `${statusCode?.toString()} ${statusText}` : statusCode?.toString();
|
|
299
|
+
const statusInfoText = statusCode >= 400 ? colors.red(`[${statusInfo}]`) : colors.green(`[${statusInfo}]`);
|
|
189
300
|
const req = res.request();
|
|
190
|
-
const duration = req.timing().responseEnd - req.timing().requestStart;
|
|
191
|
-
const durationText =
|
|
301
|
+
const duration = Math.round(req.timing().responseEnd - req.timing().requestStart);
|
|
302
|
+
const durationText = `${COLORFUL_SYMBOLS.line} ${getFormattedDuration(duration)} ${COLORFUL_SYMBOLS.line}`;
|
|
192
303
|
return {
|
|
304
|
+
/**
|
|
305
|
+
* 四舍五入 ms
|
|
306
|
+
*/
|
|
193
307
|
duration,
|
|
194
308
|
durationText,
|
|
195
309
|
statusCode,
|
|
196
310
|
statusText,
|
|
197
311
|
statusInfo,
|
|
312
|
+
statusInfoText,
|
|
198
313
|
linkText: colors.dim(res.url())
|
|
199
314
|
};
|
|
200
315
|
}
|
|
@@ -336,92 +451,62 @@ function getMEODPUrlItemInfo(urlItem) {
|
|
|
336
451
|
};
|
|
337
452
|
}
|
|
338
453
|
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
const content = formatArgs(args);
|
|
367
|
-
consola.info(content);
|
|
368
|
-
MEODP.logger._log(COLORFUL_SYMBOLS.info, content);
|
|
369
|
-
},
|
|
370
|
-
warn: (...args) => {
|
|
371
|
-
const content = formatArgs(args);
|
|
372
|
-
consola.warn(content);
|
|
373
|
-
MEODP.logger._log(COLORFUL_SYMBOLS.warn, content);
|
|
374
|
-
},
|
|
375
|
-
error: (...args) => {
|
|
376
|
-
const content = formatArgs(args);
|
|
377
|
-
consola.log(COLORFUL_SYMBOLS.error, content);
|
|
378
|
-
MEODP.logger._log(COLORFUL_SYMBOLS.error, content);
|
|
379
|
-
MEODP.wLogger.error(content);
|
|
454
|
+
const defaultMEODPConfig = {
|
|
455
|
+
urls: [],
|
|
456
|
+
concurrency: 10,
|
|
457
|
+
log: {
|
|
458
|
+
type: "raw",
|
|
459
|
+
file: true
|
|
460
|
+
}
|
|
461
|
+
};
|
|
462
|
+
function defineConfig(config) {
|
|
463
|
+
return config;
|
|
464
|
+
}
|
|
465
|
+
async function runByConfig(config) {
|
|
466
|
+
const { urls, debug } = config;
|
|
467
|
+
consola.level = debug ? 5 : 3;
|
|
468
|
+
const queue = new PQueue({
|
|
469
|
+
concurrency: config.concurrency
|
|
470
|
+
});
|
|
471
|
+
if (config.log?.type === "progress") {
|
|
472
|
+
for (const urlItem of urls) {
|
|
473
|
+
const { type, url, emoji } = getMEODPUrlItemInfo(urlItem);
|
|
474
|
+
const bar = multiBar.create(1, 0, {
|
|
475
|
+
emoji,
|
|
476
|
+
url,
|
|
477
|
+
type,
|
|
478
|
+
error_count: 0
|
|
479
|
+
});
|
|
480
|
+
progressBarMap.set(url, bar);
|
|
380
481
|
}
|
|
381
|
-
|
|
482
|
+
const curBar = multiBar.create(1, 0, {
|
|
483
|
+
name: "CURRENT"
|
|
484
|
+
});
|
|
485
|
+
progressBarMap.set("CURRENT", curBar);
|
|
486
|
+
}
|
|
487
|
+
const browser = await getBrowser();
|
|
488
|
+
for (const urlItem of urls) {
|
|
489
|
+
await queue.add(async () => {
|
|
490
|
+
await checkMEODPUrl(urlItem);
|
|
491
|
+
});
|
|
492
|
+
}
|
|
493
|
+
await queue.onIdle();
|
|
494
|
+
if (config.log?.type === "progress") {
|
|
495
|
+
multiBar.stop();
|
|
496
|
+
}
|
|
497
|
+
await browser.close();
|
|
498
|
+
consola.debug("browser closed");
|
|
499
|
+
return true;
|
|
382
500
|
}
|
|
383
501
|
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
// winston.format.errors({ stack: true }),
|
|
393
|
-
// winston.format.json(),
|
|
394
|
-
// winston.format.simple(),
|
|
395
|
-
// winston.format.uncolorize(),
|
|
396
|
-
];
|
|
397
|
-
customFormats.push(
|
|
398
|
-
winston.format.printf(({ level, message, timestamp, service }) => {
|
|
399
|
-
const content = [
|
|
400
|
-
timestamp,
|
|
401
|
-
`[${service}]`,
|
|
402
|
-
level.toUpperCase(),
|
|
403
|
-
message
|
|
404
|
-
];
|
|
405
|
-
return content.join(" ");
|
|
406
|
-
})
|
|
407
|
-
);
|
|
408
|
-
const customTransports = [
|
|
409
|
-
new winston.transports.File({
|
|
410
|
-
filename: errorLogPath,
|
|
411
|
-
level: "error"
|
|
412
|
-
}),
|
|
413
|
-
new winston.transports.File({
|
|
414
|
-
filename: allLogPath
|
|
415
|
-
// level: 'info',
|
|
416
|
-
})
|
|
417
|
-
];
|
|
418
|
-
return winston.createLogger({
|
|
419
|
-
level: "info",
|
|
420
|
-
format: winston.format.combine(...customFormats),
|
|
421
|
-
defaultMeta: { service: "MEODP" },
|
|
422
|
-
transports: customTransports,
|
|
423
|
-
exitOnError: false
|
|
424
|
-
});
|
|
502
|
+
const defaultData = {
|
|
503
|
+
config: defaultMEODPConfig,
|
|
504
|
+
sites: {}
|
|
505
|
+
};
|
|
506
|
+
async function createLowDB(rootDir = process.cwd()) {
|
|
507
|
+
const dbJsonPath = path.resolve(rootDir, "logs/meodp/db.json");
|
|
508
|
+
const db = await JSONFilePreset(dbJsonPath, defaultData);
|
|
509
|
+
return db;
|
|
425
510
|
}
|
|
426
511
|
|
|
427
512
|
var __defProp = Object.defineProperty;
|
|
@@ -446,6 +531,9 @@ const _MEODP = class _MEODP {
|
|
|
446
531
|
await LocalLog.init();
|
|
447
532
|
_MEODP.logFolder = LocalLog.logFolder || path.resolve(process.cwd(), "logs/meodp");
|
|
448
533
|
_MEODP.wLogger = createWinstonLogger();
|
|
534
|
+
const db = await createLowDB();
|
|
535
|
+
_MEODP.db = db;
|
|
536
|
+
await db.update((data) => data.config = _MEODP.config);
|
|
449
537
|
}
|
|
450
538
|
/**
|
|
451
539
|
* filter link by ignoreLinks
|
|
@@ -494,6 +582,7 @@ __publicField(_MEODP, "browser");
|
|
|
494
582
|
__publicField(_MEODP, "logFolder");
|
|
495
583
|
__publicField(_MEODP, "logger", createMEODPLogger());
|
|
496
584
|
__publicField(_MEODP, "wLogger");
|
|
585
|
+
__publicField(_MEODP, "db");
|
|
497
586
|
let MEODP = _MEODP;
|
|
498
587
|
|
|
499
588
|
async function getBrowser() {
|
|
@@ -542,7 +631,6 @@ async function checkUrlNomoduleAssets(page) {
|
|
|
542
631
|
if (!nomoduleUrl) {
|
|
543
632
|
return;
|
|
544
633
|
}
|
|
545
|
-
const startTime = Date.now();
|
|
546
634
|
const newPage = await context.newPage();
|
|
547
635
|
const response = await newPage.goto(nomoduleUrl);
|
|
548
636
|
if (!response) {
|
|
@@ -553,7 +641,8 @@ async function checkUrlNomoduleAssets(page) {
|
|
|
553
641
|
statusInfo,
|
|
554
642
|
linkText
|
|
555
643
|
} = getFormattedDataFromResponse(response);
|
|
556
|
-
const
|
|
644
|
+
const req = response.request();
|
|
645
|
+
const duration = req.timing().responseEnd - req.timing().requestStart;
|
|
557
646
|
const durationTxt = colors.dim(`\u2502${colors.blue(`${duration.toString().padStart(4, " ")}${colors.white("ms")}`)} \u2502`);
|
|
558
647
|
if (statusCode >= 400) {
|
|
559
648
|
MEODP.logger.inner(" ", COLORFUL_SYMBOLS.error, colors.red(statusInfo), durationTxt, linkText);
|
|
@@ -571,9 +660,31 @@ async function checkSiteUrl(url, options) {
|
|
|
571
660
|
const browser = await getBrowser();
|
|
572
661
|
const page = await browser.newPage();
|
|
573
662
|
const bar = progressBarMap.get(options.urlItem.url);
|
|
574
|
-
const startTime = Date.now();
|
|
575
663
|
const isExternalLink = MEODP.isExternalLink(url, options.urlItem);
|
|
576
664
|
const log = LocalLog.createLog(options.urlItem);
|
|
665
|
+
const siteData = MEODP.db.data.sites[options.urlItem.url];
|
|
666
|
+
const siteLinkItem = {
|
|
667
|
+
url,
|
|
668
|
+
statusCode: 0,
|
|
669
|
+
checkStatus: "pending",
|
|
670
|
+
success: 0,
|
|
671
|
+
failed: 0,
|
|
672
|
+
ignored: 0,
|
|
673
|
+
timeout: 0,
|
|
674
|
+
duration: 0,
|
|
675
|
+
total: 0,
|
|
676
|
+
requests: []
|
|
677
|
+
};
|
|
678
|
+
if (MEODP.isIgnoredLink(url)) {
|
|
679
|
+
siteUrlMap.set(url, {
|
|
680
|
+
statusCode: 0,
|
|
681
|
+
checkStatus: "ignored"
|
|
682
|
+
});
|
|
683
|
+
MEODP.logger.inner(`${colors.dim(`[Ignored] ${colors.underline(url)}`)}`);
|
|
684
|
+
siteData.ignored.push(url);
|
|
685
|
+
await MEODP.db.write();
|
|
686
|
+
return;
|
|
687
|
+
}
|
|
577
688
|
if (!MEODP.config.checkExternalLinks && isExternalLink) {
|
|
578
689
|
let res = null;
|
|
579
690
|
try {
|
|
@@ -581,18 +692,36 @@ async function checkSiteUrl(url, options) {
|
|
|
581
692
|
waitUntil: "load"
|
|
582
693
|
});
|
|
583
694
|
} catch (e) {
|
|
695
|
+
siteLinkItem.timeout = 1;
|
|
584
696
|
MEODP.logger.error(e);
|
|
585
697
|
}
|
|
586
698
|
if (!res)
|
|
587
699
|
return;
|
|
588
|
-
const
|
|
700
|
+
const {
|
|
701
|
+
statusCode,
|
|
702
|
+
statusInfoText,
|
|
703
|
+
duration,
|
|
704
|
+
durationText
|
|
705
|
+
} = getFormattedDataFromResponse(res);
|
|
589
706
|
const checkStatus = statusCode < 400 ? "passed" : "failed";
|
|
590
707
|
siteUrlMap.set(url, {
|
|
591
708
|
response: res,
|
|
592
709
|
statusCode,
|
|
593
710
|
checkStatus
|
|
594
711
|
});
|
|
595
|
-
MEODP.logger.
|
|
712
|
+
MEODP.logger.inner(statusInfoText, colors.cyan(url), durationText);
|
|
713
|
+
if (statusCode >= 400) {
|
|
714
|
+
MEODP.wLogger.error(` ${statusInfoText} ${url} ${durationText}`);
|
|
715
|
+
}
|
|
716
|
+
siteLinkItem.statusCode = statusCode;
|
|
717
|
+
siteLinkItem.checkStatus = checkStatus;
|
|
718
|
+
siteLinkItem.duration = duration;
|
|
719
|
+
siteLinkItem.total = 1;
|
|
720
|
+
siteLinkItem.success = checkStatus === "passed" ? 1 : 0;
|
|
721
|
+
siteLinkItem.failed = checkStatus === "failed" ? 1 : 0;
|
|
722
|
+
siteLinkItem.requests = [];
|
|
723
|
+
siteData.links.push(siteLinkItem);
|
|
724
|
+
await MEODP.db.write();
|
|
596
725
|
return;
|
|
597
726
|
}
|
|
598
727
|
const urlMap = registerPageEvents(page, options.urlItem);
|
|
@@ -601,20 +730,41 @@ async function checkSiteUrl(url, options) {
|
|
|
601
730
|
waitUntil: "networkidle"
|
|
602
731
|
});
|
|
603
732
|
const statusCode = res?.status() || 0;
|
|
733
|
+
const statusText = res?.statusText();
|
|
734
|
+
const req = res?.request();
|
|
735
|
+
const duration = Math.round((req?.timing()?.responseEnd || 0) - (req?.timing()?.requestStart || 0));
|
|
604
736
|
siteUrlMap.set(url, {
|
|
605
737
|
response: res,
|
|
606
738
|
statusCode,
|
|
607
739
|
checkStatus: "goto"
|
|
608
740
|
});
|
|
609
741
|
const { success, failed, total, ignored, timeout } = parseUrlMap(urlMap);
|
|
610
|
-
|
|
611
|
-
|
|
742
|
+
siteLinkItem.title = await page.title();
|
|
743
|
+
siteLinkItem.statusCode = statusCode;
|
|
744
|
+
siteLinkItem.statusText = statusText;
|
|
745
|
+
siteLinkItem.checkStatus = "goto";
|
|
746
|
+
siteLinkItem.duration = duration;
|
|
747
|
+
siteLinkItem.total = total.count;
|
|
748
|
+
siteLinkItem.success = success.count;
|
|
749
|
+
siteLinkItem.failed = failed.count;
|
|
750
|
+
siteLinkItem.timeout = timeout.count;
|
|
751
|
+
siteLinkItem.ignored = ignored.count;
|
|
752
|
+
siteLinkItem.requests = Array.from(urlMap.values()).map((value) => {
|
|
753
|
+
const res2 = value.response;
|
|
754
|
+
return {
|
|
755
|
+
failed: value.failed,
|
|
756
|
+
ignored: value.ignored,
|
|
757
|
+
url: res2?.url(),
|
|
758
|
+
statusCode: res2?.status(),
|
|
759
|
+
statusText: res2?.statusText()
|
|
760
|
+
};
|
|
761
|
+
});
|
|
612
762
|
const logInfo = [
|
|
613
763
|
COLORFUL_SYMBOLS.line,
|
|
614
764
|
" ",
|
|
615
765
|
colors.green(`[${statusCode}${statusText ? ` ${statusText}` : ""}]`),
|
|
616
766
|
colors.cyan(url),
|
|
617
|
-
colors.dim(`(in ${duration}
|
|
767
|
+
colors.dim(`(in ${duration}ms)`),
|
|
618
768
|
total.text,
|
|
619
769
|
success.text,
|
|
620
770
|
failed.text,
|
|
@@ -624,36 +774,41 @@ async function checkSiteUrl(url, options) {
|
|
|
624
774
|
MEODP.logger.log(...logInfo);
|
|
625
775
|
log(...logInfo);
|
|
626
776
|
log();
|
|
777
|
+
if (failed.count > 0) {
|
|
778
|
+
MEODP.wLogger.error(logInfo.join(" "));
|
|
779
|
+
}
|
|
627
780
|
if (MEODP.config.log?.type === "progress") {
|
|
628
781
|
bar?.update(success.count, {
|
|
629
782
|
value: colors.green(success.count),
|
|
630
783
|
error_count: failed.count ? colors.red(failed.count) : 0
|
|
631
784
|
});
|
|
632
|
-
} else {
|
|
633
785
|
}
|
|
634
786
|
} catch (e) {
|
|
635
787
|
MEODP.logger.error(e);
|
|
636
788
|
}
|
|
637
789
|
for (const [url2, info] of urlMap) {
|
|
638
790
|
if (!info.response && !info.ignored) {
|
|
639
|
-
MEODP.logger.log(COLORFUL_SYMBOLS.line, " ", COLORFUL_SYMBOLS.error,
|
|
791
|
+
MEODP.logger.log(COLORFUL_SYMBOLS.line, " ", COLORFUL_SYMBOLS.error, colors.red("Timeout:"), colors.underline(url2));
|
|
792
|
+
MEODP.wLogger.error(` Timeout: ${url2}`);
|
|
793
|
+
siteLinkItem.checkStatus = "timeout";
|
|
640
794
|
}
|
|
641
795
|
}
|
|
642
796
|
let noModuleAssetsPassed = true;
|
|
643
797
|
if (options.checkNoModule) {
|
|
644
798
|
noModuleAssetsPassed = await checkUrlNomoduleAssets(page);
|
|
645
799
|
}
|
|
646
|
-
const urlMapValues = Array.from(urlMap.values());
|
|
647
800
|
const siteUrlItem = siteUrlMap.get(url);
|
|
648
801
|
if (siteUrlItem) {
|
|
649
|
-
|
|
802
|
+
const isAllRequestsPassed = siteLinkItem.requests.filter((value) => !value.ignored).every((value) => value.statusCode && value.statusCode < 400);
|
|
803
|
+
if (isAllRequestsPassed && noModuleAssetsPassed) {
|
|
650
804
|
siteUrlItem.checkStatus = "passed";
|
|
651
|
-
const successCount = Array.from(siteUrlMap.values()).filter((value) => value.checkStatus === "passed").length;
|
|
652
|
-
bar?.update(successCount);
|
|
653
805
|
} else {
|
|
654
806
|
siteUrlItem.checkStatus = "failed";
|
|
655
807
|
}
|
|
808
|
+
siteLinkItem.checkStatus = siteUrlItem.checkStatus;
|
|
656
809
|
}
|
|
810
|
+
siteData.links.push(siteLinkItem);
|
|
811
|
+
await MEODP.db.write();
|
|
657
812
|
if (!isExternalLink) {
|
|
658
813
|
const queue = PQueueMap.get(url);
|
|
659
814
|
if (queue) {
|
|
@@ -674,6 +829,7 @@ async function checkSiteUrl(url, options) {
|
|
|
674
829
|
}
|
|
675
830
|
}
|
|
676
831
|
await page.close();
|
|
832
|
+
return siteLinkItem;
|
|
677
833
|
}
|
|
678
834
|
async function checkSite(props) {
|
|
679
835
|
const { url } = props;
|
|
@@ -690,11 +846,25 @@ async function checkSite(props) {
|
|
|
690
846
|
statusCode: 0,
|
|
691
847
|
checkStatus: "pending"
|
|
692
848
|
});
|
|
849
|
+
await MEODP.db.update((data) => {
|
|
850
|
+
data.sites[homeUrl] = {
|
|
851
|
+
name: props.name,
|
|
852
|
+
url: homeUrl,
|
|
853
|
+
checkStatus: "pending",
|
|
854
|
+
links: [],
|
|
855
|
+
ignored: []
|
|
856
|
+
};
|
|
857
|
+
});
|
|
693
858
|
bar?.setTotal(siteUrlMap.size);
|
|
694
|
-
await checkSiteUrl(homeUrl, {
|
|
859
|
+
const siteLinkItem = await checkSiteUrl(homeUrl, {
|
|
695
860
|
urlItem: props,
|
|
696
861
|
checkNoModule: true
|
|
697
862
|
});
|
|
863
|
+
await MEODP.db.update((data) => {
|
|
864
|
+
const siteItem = data.sites[homeUrl];
|
|
865
|
+
siteItem.title = siteLinkItem?.title;
|
|
866
|
+
siteItem.checkStatus = siteItem.links.every((link) => link.checkStatus === "passed") ? "passed" : "failed";
|
|
867
|
+
});
|
|
698
868
|
await queue.onIdle();
|
|
699
869
|
await page.close();
|
|
700
870
|
await context.close();
|
|
@@ -764,55 +934,7 @@ async function checkMEODPUrl(meodpUrl) {
|
|
|
764
934
|
const duration = Date.now() - startTime;
|
|
765
935
|
const bar = progressBarMap.get(meodpUrl.url);
|
|
766
936
|
bar?.stop();
|
|
767
|
-
MEODP.logger.success(
|
|
768
|
-
}
|
|
769
|
-
|
|
770
|
-
const defaultMEODPConfig = {
|
|
771
|
-
urls: [],
|
|
772
|
-
concurrency: 10,
|
|
773
|
-
log: {
|
|
774
|
-
type: "raw",
|
|
775
|
-
file: true
|
|
776
|
-
}
|
|
777
|
-
};
|
|
778
|
-
function defineConfig(config) {
|
|
779
|
-
return config;
|
|
780
|
-
}
|
|
781
|
-
async function runByConfig(config) {
|
|
782
|
-
const { urls, debug } = config;
|
|
783
|
-
consola.level = debug ? 5 : 3;
|
|
784
|
-
const queue = new PQueue({
|
|
785
|
-
concurrency: config.concurrency
|
|
786
|
-
});
|
|
787
|
-
if (config.log?.type === "progress") {
|
|
788
|
-
for (const urlItem of urls) {
|
|
789
|
-
const { type, url, emoji } = getMEODPUrlItemInfo(urlItem);
|
|
790
|
-
const bar = multiBar.create(1, 0, {
|
|
791
|
-
emoji,
|
|
792
|
-
url,
|
|
793
|
-
type,
|
|
794
|
-
error_count: 0
|
|
795
|
-
});
|
|
796
|
-
progressBarMap.set(url, bar);
|
|
797
|
-
}
|
|
798
|
-
const curBar = multiBar.create(1, 0, {
|
|
799
|
-
name: "CURRENT"
|
|
800
|
-
});
|
|
801
|
-
progressBarMap.set("CURRENT", curBar);
|
|
802
|
-
}
|
|
803
|
-
const browser = await getBrowser();
|
|
804
|
-
for (const urlItem of urls) {
|
|
805
|
-
await queue.add(async () => {
|
|
806
|
-
await checkMEODPUrl(urlItem);
|
|
807
|
-
});
|
|
808
|
-
}
|
|
809
|
-
await queue.onIdle();
|
|
810
|
-
if (config.log?.type === "progress") {
|
|
811
|
-
multiBar.stop();
|
|
812
|
-
}
|
|
813
|
-
await browser.close();
|
|
814
|
-
consola.debug("browser closed");
|
|
815
|
-
return true;
|
|
937
|
+
MEODP.logger.success(`\u{1F577}\uFE0F ${colors.greenBright("[DONE]")} ${colors.gray("in")} ${duration / 1e3}s.`);
|
|
816
938
|
}
|
|
817
939
|
|
|
818
|
-
export { LocalLog as L, MEODP as M, PQueueMap as P, checkLink as a, getSiteLinks as b, checkMEODPUrl as c, checkUrlNomoduleAssets as d, checkSiteUrl as e, checkSite as f, getBrowser as g, defaultMEODPConfig as h, defineConfig as i,
|
|
940
|
+
export { LocalLog as L, MEODP as M, PQueueMap as P, checkLink as a, getSiteLinks as b, checkMEODPUrl as c, checkUrlNomoduleAssets as d, checkSiteUrl as e, checkSite as f, getBrowser as g, defaultMEODPConfig as h, defineConfig as i, createLowDB as j, localLog as k, logUrlItemInfo as l, createMEODPLogger as m, createWinstonLogger as n, multiBar as o, progressBarMap as p, registerPageEvents as q, runByConfig as r, siteUrlMap as s, formatArgs as t, getFormattedDuration as u, getFormattedDataFromResponse as v, isLink as w, parseUrlMap as x, emojiMap as y, getMEODPUrlItemInfo as z };
|