meodp 0.0.6 → 0.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,37 +1,73 @@
1
1
  'use strict';
2
2
 
3
- const consola = require('consola');
4
- const PQueue = require('p-queue');
5
3
  const utils = require('consola/utils');
6
4
  const playwright = require('playwright');
7
5
  const path = require('node:path');
8
6
  const process = require('node:process');
9
- const dateFns = require('date-fns');
10
- const fs = require('fs-extra');
11
- const stripAnsi = require('strip-ansi');
12
- const winston = require('winston');
7
+ const consola = require('consola');
8
+ const node = require('lowdb/node');
9
+ const PQueue = require('p-queue');
13
10
  const cliProgress = require('cli-progress');
14
11
  const Options = require('cli-progress/lib/options.js');
15
12
  const _defaultFormatBar = require('cli-progress/lib/format-bar.js');
16
13
  const _defaultFormatTime = require('cli-progress/lib/format-time.js');
14
+ const dateFns = require('date-fns');
15
+ const fs = require('fs-extra');
16
+ const stripAnsi = require('strip-ansi');
17
+ const winston = require('winston');
17
18
 
18
19
  function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e.default : e; }
19
20
 
20
- const consola__default = /*#__PURE__*/_interopDefaultCompat(consola);
21
- const PQueue__default = /*#__PURE__*/_interopDefaultCompat(PQueue);
22
21
  const path__default = /*#__PURE__*/_interopDefaultCompat(path);
23
22
  const process__default = /*#__PURE__*/_interopDefaultCompat(process);
24
- const fs__default = /*#__PURE__*/_interopDefaultCompat(fs);
25
- const stripAnsi__default = /*#__PURE__*/_interopDefaultCompat(stripAnsi);
26
- const winston__default = /*#__PURE__*/_interopDefaultCompat(winston);
23
+ const consola__default = /*#__PURE__*/_interopDefaultCompat(consola);
24
+ const PQueue__default = /*#__PURE__*/_interopDefaultCompat(PQueue);
27
25
  const cliProgress__default = /*#__PURE__*/_interopDefaultCompat(cliProgress);
28
26
  const Options__default = /*#__PURE__*/_interopDefaultCompat(Options);
29
27
  const _defaultFormatBar__default = /*#__PURE__*/_interopDefaultCompat(_defaultFormatBar);
30
28
  const _defaultFormatTime__default = /*#__PURE__*/_interopDefaultCompat(_defaultFormatTime);
29
+ const fs__default = /*#__PURE__*/_interopDefaultCompat(fs);
30
+ const stripAnsi__default = /*#__PURE__*/_interopDefaultCompat(stripAnsi);
31
+ const winston__default = /*#__PURE__*/_interopDefaultCompat(winston);
31
32
 
32
33
  const siteUrlMap = /* @__PURE__ */ new Map();
33
34
  const PQueueMap = /* @__PURE__ */ new Map();
34
35
 
36
+ const formatter = (options, params, payload) => {
37
+ const { value, total } = params;
38
+ const formatTime = options.formatTime || _defaultFormatTime__default;
39
+ const formatBar = options.formatBar || _defaultFormatBar__default;
40
+ const bar = formatBar(params.progress, options);
41
+ const stopTime = params.stopTime || Date.now();
42
+ const elapsedTime = Math.round((stopTime - params.startTime) / 1e3);
43
+ const duration_formatted = formatTime(elapsedTime, options, 1);
44
+ consola__default.debug(duration_formatted);
45
+ if (!payload.type) {
46
+ options = Object.assign({}, options, cliProgress__default.Presets.rect);
47
+ Options__default.assignDerivedOptions(options);
48
+ const bar2 = formatBar(params.progress, options);
49
+ const { site } = payload;
50
+ const url = payload.url || "";
51
+ return `${bar2} ${value}/${total} | ${utils.colors.yellow(site || "MEODP Item")} | ${utils.colors.cyan(url || "URL")}`;
52
+ } else {
53
+ const { error_count, emoji, url, type } = payload;
54
+ const errorInfo = error_count ? `\u274C${error_count} ` : "";
55
+ return `${bar} \u2705${utils.colors.green(value)}/${total} ${errorInfo}| ${duration_formatted} | ${emoji} ${utils.colors.yellow(type)} | ${utils.colors.cyan(url)}`;
56
+ }
57
+ };
58
+ const multiBar = new cliProgress__default.MultiBar({
59
+ hideCursor: true,
60
+ // format: '{bar} {percentage}% | ✅{value}/{total} ❌{error_count} | {duration_formatted} | {name}',
61
+ format: formatter,
62
+ barsize: 20,
63
+ autopadding: true,
64
+ barCompleteChar: "\u2588",
65
+ barIncompleteChar: "\u2591",
66
+ // important! redraw everything to avoid "empty" completed bars
67
+ forceRedraw: true
68
+ }, cliProgress__default.Presets.rect);
69
+ const progressBarMap = /* @__PURE__ */ new Map();
70
+
35
71
  var __defProp$1 = Object.defineProperty;
36
72
  var __defNormalProp$1 = (obj, key, value) => key in obj ? __defProp$1(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
37
73
  var __publicField$1 = (obj, key, value) => {
@@ -164,40 +200,93 @@ const COLORFUL_SYMBOLS = {
164
200
  line: utils.colors.dim(RAW_SYMBOLS.line)
165
201
  };
166
202
 
167
- const formatter = (options, params, payload) => {
168
- const { value, total } = params;
169
- const formatTime = options.formatTime || _defaultFormatTime__default;
170
- const formatBar = options.formatBar || _defaultFormatBar__default;
171
- const bar = formatBar(params.progress, options);
172
- const stopTime = params.stopTime || Date.now();
173
- const elapsedTime = Math.round((stopTime - params.startTime) / 1e3);
174
- const duration_formatted = formatTime(elapsedTime, options, 1);
175
- consola__default.debug(duration_formatted);
176
- if (!payload.type) {
177
- options = Object.assign({}, options, cliProgress__default.Presets.rect);
178
- Options__default.assignDerivedOptions(options);
179
- const bar2 = formatBar(params.progress, options);
180
- const { site } = payload;
181
- const url = payload.url || "";
182
- return `${bar2} ${value}/${total} | ${utils.colors.yellow(site || "MEODP Item")} | ${utils.colors.cyan(url || "URL")}`;
183
- } else {
184
- const { error_count, emoji, url, type } = payload;
185
- const errorInfo = error_count ? `\u274C${error_count} ` : "";
186
- return `${bar} \u2705${utils.colors.green(value)}/${total} ${errorInfo}| ${duration_formatted} | ${emoji} ${utils.colors.yellow(type)} | ${utils.colors.cyan(url)}`;
187
- }
188
- };
189
- const multiBar = new cliProgress__default.MultiBar({
190
- hideCursor: true,
191
- // format: '{bar} {percentage}% | ✅{value}/{total} ❌{error_count} | {duration_formatted} | {name}',
192
- format: formatter,
193
- barsize: 20,
194
- autopadding: true,
195
- barCompleteChar: "\u2588",
196
- barIncompleteChar: "\u2591",
197
- // important! redraw everything to avoid "empty" completed bars
198
- forceRedraw: true
199
- }, cliProgress__default.Presets.rect);
200
- const progressBarMap = /* @__PURE__ */ new Map();
203
+ function createMEODPLogger() {
204
+ return {
205
+ _log: (...args) => {
206
+ const allLogPath = path__default.resolve(MEODP.logFolder, "all.log");
207
+ LocalLog.log(allLogPath, stripAnsi__default(args.join(" ")));
208
+ },
209
+ log: (...args) => {
210
+ const content = formatArgs(args);
211
+ consola__default.log(content);
212
+ MEODP.logger._log(content);
213
+ },
214
+ inner: (...args) => {
215
+ const content = formatArgs(args);
216
+ consola__default.log(COLORFUL_SYMBOLS.line, " ", content);
217
+ MEODP.logger._log(COLORFUL_SYMBOLS.line, " ", content);
218
+ },
219
+ start: (...args) => {
220
+ const content = formatArgs(args);
221
+ consola__default.start(content);
222
+ MEODP.logger._log(COLORFUL_SYMBOLS.start, content);
223
+ },
224
+ success: (...args) => {
225
+ const content = formatArgs(args);
226
+ consola__default.success(content);
227
+ MEODP.logger._log(COLORFUL_SYMBOLS.success, content);
228
+ },
229
+ info: (...args) => {
230
+ const content = formatArgs(args);
231
+ consola__default.info(content);
232
+ MEODP.logger._log(COLORFUL_SYMBOLS.info, content);
233
+ },
234
+ warn: (...args) => {
235
+ const content = formatArgs(args);
236
+ consola__default.warn(content);
237
+ MEODP.logger._log(COLORFUL_SYMBOLS.warn, content);
238
+ },
239
+ error: (...args) => {
240
+ const content = formatArgs(args);
241
+ consola__default.log(COLORFUL_SYMBOLS.error, content);
242
+ MEODP.logger._log(COLORFUL_SYMBOLS.error, content);
243
+ MEODP.wLogger.error(content);
244
+ }
245
+ };
246
+ }
247
+
248
+ function createWinstonLogger() {
249
+ const allLogPath = path__default.resolve(MEODP.logFolder, "all.log");
250
+ const errorLogPath = path__default.resolve(MEODP.logFolder, "error.log");
251
+ const customFormats = [
252
+ winston__default.format.timestamp({
253
+ format: "YYYY-MM-DD HH:mm:ss"
254
+ })
255
+ // winston.format.colorize(),
256
+ // winston.format.errors({ stack: true }),
257
+ // winston.format.json(),
258
+ // winston.format.simple(),
259
+ // winston.format.uncolorize(),
260
+ ];
261
+ customFormats.push(
262
+ winston__default.format.printf(({ level, message, timestamp, service }) => {
263
+ const content = [
264
+ timestamp,
265
+ `[${service}]`,
266
+ level.toUpperCase(),
267
+ message
268
+ ];
269
+ return content.join(" ");
270
+ })
271
+ );
272
+ const customTransports = [
273
+ new winston__default.transports.File({
274
+ filename: errorLogPath,
275
+ level: "error"
276
+ }),
277
+ new winston__default.transports.File({
278
+ filename: allLogPath
279
+ // level: 'info',
280
+ })
281
+ ];
282
+ return winston__default.createLogger({
283
+ level: "info",
284
+ format: winston__default.format.combine(...customFormats),
285
+ defaultMeta: { service: "MEODP" },
286
+ transports: customTransports,
287
+ exitOnError: false
288
+ });
289
+ }
201
290
 
202
291
  function formatArgs(args) {
203
292
  return args.join(" ");
@@ -208,7 +297,7 @@ const TIME_COLOR = {
208
297
  slow: utils.colors.red
209
298
  };
210
299
  function getFormattedDuration(duration) {
211
- const durationText = `${(Math.round(duration) / 1e3).toString()}s`;
300
+ const durationText = `${(duration / 1e3).toString()}s`;
212
301
  let type;
213
302
  if (duration < 500) {
214
303
  type = "fast";
@@ -225,9 +314,12 @@ function getFormattedDataFromResponse(res) {
225
314
  const statusInfo = statusText ? `${statusCode?.toString()} ${statusText}` : statusCode?.toString();
226
315
  const statusInfoText = statusCode >= 400 ? utils.colors.red(`[${statusInfo}]`) : utils.colors.green(`[${statusInfo}]`);
227
316
  const req = res.request();
228
- const duration = req.timing().responseEnd - req.timing().requestStart;
317
+ const duration = Math.round(req.timing().responseEnd - req.timing().requestStart);
229
318
  const durationText = `${COLORFUL_SYMBOLS.line} ${getFormattedDuration(duration)} ${COLORFUL_SYMBOLS.line}`;
230
319
  return {
320
+ /**
321
+ * 四舍五入 ms
322
+ */
231
323
  duration,
232
324
  durationText,
233
325
  statusCode,
@@ -375,92 +467,62 @@ function getMEODPUrlItemInfo(urlItem) {
375
467
  };
376
468
  }
377
469
 
378
- function createMEODPLogger() {
379
- return {
380
- _log: (...args) => {
381
- const allLogPath = path__default.resolve(MEODP.logFolder, "all.log");
382
- LocalLog.log(allLogPath, stripAnsi__default(args.join(" ")));
383
- },
384
- log: (...args) => {
385
- const content = formatArgs(args);
386
- consola__default.log(content);
387
- MEODP.logger._log(content);
388
- },
389
- inner: (...args) => {
390
- const content = formatArgs(args);
391
- consola__default.log(COLORFUL_SYMBOLS.line, " ", content);
392
- MEODP.logger._log(COLORFUL_SYMBOLS.line, " ", content);
393
- },
394
- start: (...args) => {
395
- const content = formatArgs(args);
396
- consola__default.start(content);
397
- MEODP.logger._log(COLORFUL_SYMBOLS.start, content);
398
- },
399
- success: (...args) => {
400
- const content = formatArgs(args);
401
- consola__default.success(content);
402
- MEODP.logger._log(COLORFUL_SYMBOLS.success, content);
403
- },
404
- info: (...args) => {
405
- const content = formatArgs(args);
406
- consola__default.info(content);
407
- MEODP.logger._log(COLORFUL_SYMBOLS.info, content);
408
- },
409
- warn: (...args) => {
410
- const content = formatArgs(args);
411
- consola__default.warn(content);
412
- MEODP.logger._log(COLORFUL_SYMBOLS.warn, content);
413
- },
414
- error: (...args) => {
415
- const content = formatArgs(args);
416
- consola__default.log(COLORFUL_SYMBOLS.error, content);
417
- MEODP.logger._log(COLORFUL_SYMBOLS.error, content);
418
- MEODP.wLogger.error(content);
470
+ const defaultMEODPConfig = {
471
+ urls: [],
472
+ concurrency: 10,
473
+ log: {
474
+ type: "raw",
475
+ file: true
476
+ }
477
+ };
478
+ function defineConfig(config) {
479
+ return config;
480
+ }
481
+ async function runByConfig(config) {
482
+ const { urls, debug } = config;
483
+ consola__default.level = debug ? 5 : 3;
484
+ const queue = new PQueue__default({
485
+ concurrency: config.concurrency
486
+ });
487
+ if (config.log?.type === "progress") {
488
+ for (const urlItem of urls) {
489
+ const { type, url, emoji } = getMEODPUrlItemInfo(urlItem);
490
+ const bar = multiBar.create(1, 0, {
491
+ emoji,
492
+ url,
493
+ type,
494
+ error_count: 0
495
+ });
496
+ progressBarMap.set(url, bar);
419
497
  }
420
- };
498
+ const curBar = multiBar.create(1, 0, {
499
+ name: "CURRENT"
500
+ });
501
+ progressBarMap.set("CURRENT", curBar);
502
+ }
503
+ const browser = await getBrowser();
504
+ for (const urlItem of urls) {
505
+ await queue.add(async () => {
506
+ await checkMEODPUrl(urlItem);
507
+ });
508
+ }
509
+ await queue.onIdle();
510
+ if (config.log?.type === "progress") {
511
+ multiBar.stop();
512
+ }
513
+ await browser.close();
514
+ consola__default.debug("browser closed");
515
+ return true;
421
516
  }
422
517
 
423
- function createWinstonLogger() {
424
- const allLogPath = path__default.resolve(MEODP.logFolder, "all.log");
425
- const errorLogPath = path__default.resolve(MEODP.logFolder, "error.log");
426
- const customFormats = [
427
- winston__default.format.timestamp({
428
- format: "YYYY-MM-DD HH:mm:ss"
429
- })
430
- // winston.format.colorize(),
431
- // winston.format.errors({ stack: true }),
432
- // winston.format.json(),
433
- // winston.format.simple(),
434
- // winston.format.uncolorize(),
435
- ];
436
- customFormats.push(
437
- winston__default.format.printf(({ level, message, timestamp, service }) => {
438
- const content = [
439
- timestamp,
440
- `[${service}]`,
441
- level.toUpperCase(),
442
- message
443
- ];
444
- return content.join(" ");
445
- })
446
- );
447
- const customTransports = [
448
- new winston__default.transports.File({
449
- filename: errorLogPath,
450
- level: "error"
451
- }),
452
- new winston__default.transports.File({
453
- filename: allLogPath
454
- // level: 'info',
455
- })
456
- ];
457
- return winston__default.createLogger({
458
- level: "info",
459
- format: winston__default.format.combine(...customFormats),
460
- defaultMeta: { service: "MEODP" },
461
- transports: customTransports,
462
- exitOnError: false
463
- });
518
+ const defaultData = {
519
+ config: defaultMEODPConfig,
520
+ sites: {}
521
+ };
522
+ async function createLowDB(rootDir = process__default.cwd()) {
523
+ const dbJsonPath = path__default.resolve(rootDir, "logs/meodp/db.json");
524
+ const db = await node.JSONFilePreset(dbJsonPath, defaultData);
525
+ return db;
464
526
  }
465
527
 
466
528
  var __defProp = Object.defineProperty;
@@ -485,6 +547,9 @@ const _MEODP = class _MEODP {
485
547
  await LocalLog.init();
486
548
  _MEODP.logFolder = LocalLog.logFolder || path__default.resolve(process__default.cwd(), "logs/meodp");
487
549
  _MEODP.wLogger = createWinstonLogger();
550
+ const db = await createLowDB();
551
+ _MEODP.db = db;
552
+ await db.update((data) => data.config = _MEODP.config);
488
553
  }
489
554
  /**
490
555
  * filter link by ignoreLinks
@@ -533,6 +598,7 @@ __publicField(_MEODP, "browser");
533
598
  __publicField(_MEODP, "logFolder");
534
599
  __publicField(_MEODP, "logger", createMEODPLogger());
535
600
  __publicField(_MEODP, "wLogger");
601
+ __publicField(_MEODP, "db");
536
602
  let MEODP = _MEODP;
537
603
 
538
604
  async function getBrowser() {
@@ -581,7 +647,6 @@ async function checkUrlNomoduleAssets(page) {
581
647
  if (!nomoduleUrl) {
582
648
  return;
583
649
  }
584
- const startTime = Date.now();
585
650
  const newPage = await context.newPage();
586
651
  const response = await newPage.goto(nomoduleUrl);
587
652
  if (!response) {
@@ -592,7 +657,8 @@ async function checkUrlNomoduleAssets(page) {
592
657
  statusInfo,
593
658
  linkText
594
659
  } = getFormattedDataFromResponse(response);
595
- const duration = Date.now() - startTime;
660
+ const req = response.request();
661
+ const duration = req.timing().responseEnd - req.timing().requestStart;
596
662
  const durationTxt = utils.colors.dim(`\u2502${utils.colors.blue(`${duration.toString().padStart(4, " ")}${utils.colors.white("ms")}`)} \u2502`);
597
663
  if (statusCode >= 400) {
598
664
  MEODP.logger.inner(" ", COLORFUL_SYMBOLS.error, utils.colors.red(statusInfo), durationTxt, linkText);
@@ -610,9 +676,31 @@ async function checkSiteUrl(url, options) {
610
676
  const browser = await getBrowser();
611
677
  const page = await browser.newPage();
612
678
  const bar = progressBarMap.get(options.urlItem.url);
613
- const startTime = Date.now();
614
679
  const isExternalLink = MEODP.isExternalLink(url, options.urlItem);
615
680
  const log = LocalLog.createLog(options.urlItem);
681
+ const siteData = MEODP.db.data.sites[options.urlItem.url];
682
+ const siteLinkItem = {
683
+ url,
684
+ statusCode: 0,
685
+ checkStatus: "pending",
686
+ success: 0,
687
+ failed: 0,
688
+ ignored: 0,
689
+ timeout: 0,
690
+ duration: 0,
691
+ total: 0,
692
+ requests: []
693
+ };
694
+ if (MEODP.isIgnoredLink(url)) {
695
+ siteUrlMap.set(url, {
696
+ statusCode: 0,
697
+ checkStatus: "ignored"
698
+ });
699
+ MEODP.logger.inner(`${utils.colors.dim(`[Ignored] ${utils.colors.underline(url)}`)}`);
700
+ siteData.ignored.push(url);
701
+ await MEODP.db.write();
702
+ return;
703
+ }
616
704
  if (!MEODP.config.checkExternalLinks && isExternalLink) {
617
705
  let res = null;
618
706
  try {
@@ -620,13 +708,19 @@ async function checkSiteUrl(url, options) {
620
708
  waitUntil: "load"
621
709
  });
622
710
  } catch (e) {
623
- MEODP.logger.error(e);
711
+ siteLinkItem.timeout = 1;
712
+ MEODP.logger.inner(
713
+ utils.colors.red("[Timeout]"),
714
+ utils.colors.underline(url)
715
+ );
716
+ consola__default.debug(e);
624
717
  }
625
718
  if (!res)
626
719
  return;
627
720
  const {
628
721
  statusCode,
629
722
  statusInfoText,
723
+ duration,
630
724
  durationText
631
725
  } = getFormattedDataFromResponse(res);
632
726
  const checkStatus = statusCode < 400 ? "passed" : "failed";
@@ -639,28 +733,59 @@ async function checkSiteUrl(url, options) {
639
733
  if (statusCode >= 400) {
640
734
  MEODP.wLogger.error(` ${statusInfoText} ${url} ${durationText}`);
641
735
  }
736
+ siteLinkItem.statusCode = statusCode;
737
+ siteLinkItem.checkStatus = checkStatus;
738
+ siteLinkItem.duration = duration;
739
+ siteLinkItem.total = 1;
740
+ siteLinkItem.success = checkStatus === "passed" ? 1 : 0;
741
+ siteLinkItem.failed = checkStatus === "failed" ? 1 : 0;
742
+ siteLinkItem.requests = [];
743
+ siteData.links.push(siteLinkItem);
744
+ await MEODP.db.write();
642
745
  return;
643
746
  }
644
747
  const urlMap = registerPageEvents(page, options.urlItem);
645
- const { success, failed, total, ignored, timeout } = parseUrlMap(urlMap);
646
748
  try {
647
749
  const res = await page.goto(url, {
648
750
  waitUntil: "networkidle"
649
751
  });
650
752
  const statusCode = res?.status() || 0;
753
+ const statusText = res?.statusText();
754
+ const req = res?.request();
755
+ const duration = Math.round((req?.timing()?.responseEnd || 0) - (req?.timing()?.requestStart || 0));
651
756
  siteUrlMap.set(url, {
652
757
  response: res,
653
758
  statusCode,
654
759
  checkStatus: "goto"
655
760
  });
656
- const duration = (Date.now() - startTime) / 1e3;
657
- const statusText = res?.statusText();
761
+ const { success, failed, total, ignored, timeout } = parseUrlMap(urlMap);
762
+ siteLinkItem.title = await page.title();
763
+ siteLinkItem.statusCode = statusCode;
764
+ siteLinkItem.statusText = statusText;
765
+ siteLinkItem.checkStatus = "goto";
766
+ siteLinkItem.duration = duration;
767
+ siteLinkItem.total = total.count;
768
+ siteLinkItem.success = success.count;
769
+ siteLinkItem.failed = failed.count;
770
+ siteLinkItem.timeout = timeout.count;
771
+ siteLinkItem.ignored = ignored.count;
772
+ siteLinkItem.requests = Array.from(urlMap.values()).map((value) => {
773
+ const res2 = value.response;
774
+ return {
775
+ failed: value.failed,
776
+ ignored: value.ignored,
777
+ // res.url 还没用返回
778
+ url: req?.url(),
779
+ statusCode: res2?.status(),
780
+ statusText: res2?.statusText()
781
+ };
782
+ });
658
783
  const logInfo = [
659
784
  COLORFUL_SYMBOLS.line,
660
785
  " ",
661
786
  utils.colors.green(`[${statusCode}${statusText ? ` ${statusText}` : ""}]`),
662
787
  utils.colors.cyan(url),
663
- utils.colors.dim(`(in ${duration}s)`),
788
+ utils.colors.dim(`(in ${duration}ms)`),
664
789
  total.text,
665
790
  success.text,
666
791
  failed.text,
@@ -686,23 +811,25 @@ async function checkSiteUrl(url, options) {
686
811
  if (!info.response && !info.ignored) {
687
812
  MEODP.logger.log(COLORFUL_SYMBOLS.line, " ", COLORFUL_SYMBOLS.error, utils.colors.red("Timeout:"), utils.colors.underline(url2));
688
813
  MEODP.wLogger.error(` Timeout: ${url2}`);
814
+ siteLinkItem.checkStatus = "timeout";
689
815
  }
690
816
  }
691
817
  let noModuleAssetsPassed = true;
692
818
  if (options.checkNoModule) {
693
819
  noModuleAssetsPassed = await checkUrlNomoduleAssets(page);
694
820
  }
695
- const urlMapValues = Array.from(urlMap.values());
696
821
  const siteUrlItem = siteUrlMap.get(url);
697
822
  if (siteUrlItem) {
698
- if (urlMapValues.every((value) => value.response?.status() && value.response?.status() < 400) && noModuleAssetsPassed) {
823
+ const isAllRequestsPassed = siteLinkItem.requests.filter((value) => !value.ignored).every((value) => value.statusCode && value.statusCode < 400);
824
+ if (isAllRequestsPassed && noModuleAssetsPassed) {
699
825
  siteUrlItem.checkStatus = "passed";
700
- const successCount = Array.from(siteUrlMap.values()).filter((value) => value.checkStatus === "passed").length;
701
- bar?.update(successCount);
702
826
  } else {
703
827
  siteUrlItem.checkStatus = "failed";
704
828
  }
829
+ siteLinkItem.checkStatus = siteUrlItem.checkStatus;
705
830
  }
831
+ siteData.links.push(siteLinkItem);
832
+ await MEODP.db.write();
706
833
  if (!isExternalLink) {
707
834
  const queue = PQueueMap.get(url);
708
835
  if (queue) {
@@ -723,6 +850,7 @@ async function checkSiteUrl(url, options) {
723
850
  }
724
851
  }
725
852
  await page.close();
853
+ return siteLinkItem;
726
854
  }
727
855
  async function checkSite(props) {
728
856
  const { url } = props;
@@ -739,11 +867,25 @@ async function checkSite(props) {
739
867
  statusCode: 0,
740
868
  checkStatus: "pending"
741
869
  });
870
+ await MEODP.db.update((data) => {
871
+ data.sites[homeUrl] = {
872
+ name: props.name,
873
+ url: homeUrl,
874
+ checkStatus: "pending",
875
+ links: [],
876
+ ignored: []
877
+ };
878
+ });
742
879
  bar?.setTotal(siteUrlMap.size);
743
- await checkSiteUrl(homeUrl, {
880
+ const siteLinkItem = await checkSiteUrl(homeUrl, {
744
881
  urlItem: props,
745
882
  checkNoModule: true
746
883
  });
884
+ await MEODP.db.update((data) => {
885
+ const siteItem = data.sites[homeUrl];
886
+ siteItem.title = siteLinkItem?.title;
887
+ siteItem.checkStatus = siteItem.links.every((link) => link.checkStatus === "passed") ? "passed" : "failed";
888
+ });
747
889
  await queue.onIdle();
748
890
  await page.close();
749
891
  await context.close();
@@ -816,54 +958,6 @@ async function checkMEODPUrl(meodpUrl) {
816
958
  MEODP.logger.success(`\u{1F577}\uFE0F ${utils.colors.greenBright("[DONE]")} ${utils.colors.gray("in")} ${duration / 1e3}s.`);
817
959
  }
818
960
 
819
- const defaultMEODPConfig = {
820
- urls: [],
821
- concurrency: 10,
822
- log: {
823
- type: "raw",
824
- file: true
825
- }
826
- };
827
- function defineConfig(config) {
828
- return config;
829
- }
830
- async function runByConfig(config) {
831
- const { urls, debug } = config;
832
- consola__default.level = debug ? 5 : 3;
833
- const queue = new PQueue__default({
834
- concurrency: config.concurrency
835
- });
836
- if (config.log?.type === "progress") {
837
- for (const urlItem of urls) {
838
- const { type, url, emoji } = getMEODPUrlItemInfo(urlItem);
839
- const bar = multiBar.create(1, 0, {
840
- emoji,
841
- url,
842
- type,
843
- error_count: 0
844
- });
845
- progressBarMap.set(url, bar);
846
- }
847
- const curBar = multiBar.create(1, 0, {
848
- name: "CURRENT"
849
- });
850
- progressBarMap.set("CURRENT", curBar);
851
- }
852
- const browser = await getBrowser();
853
- for (const urlItem of urls) {
854
- await queue.add(async () => {
855
- await checkMEODPUrl(urlItem);
856
- });
857
- }
858
- await queue.onIdle();
859
- if (config.log?.type === "progress") {
860
- multiBar.stop();
861
- }
862
- await browser.close();
863
- consola__default.debug("browser closed");
864
- return true;
865
- }
866
-
867
961
  exports.LocalLog = LocalLog;
868
962
  exports.MEODP = MEODP;
869
963
  exports.PQueueMap = PQueueMap;
@@ -872,6 +966,7 @@ exports.checkMEODPUrl = checkMEODPUrl;
872
966
  exports.checkSite = checkSite;
873
967
  exports.checkSiteUrl = checkSiteUrl;
874
968
  exports.checkUrlNomoduleAssets = checkUrlNomoduleAssets;
969
+ exports.createLowDB = createLowDB;
875
970
  exports.createMEODPLogger = createMEODPLogger;
876
971
  exports.createWinstonLogger = createWinstonLogger;
877
972
  exports.defaultMEODPConfig = defaultMEODPConfig;