meodp 0.0.6 → 0.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,37 +1,73 @@
1
1
  'use strict';
2
2
 
3
- const consola = require('consola');
4
- const PQueue = require('p-queue');
5
3
  const utils = require('consola/utils');
6
4
  const playwright = require('playwright');
7
5
  const path = require('node:path');
8
6
  const process = require('node:process');
9
- const dateFns = require('date-fns');
10
- const fs = require('fs-extra');
11
- const stripAnsi = require('strip-ansi');
12
- const winston = require('winston');
7
+ const consola = require('consola');
8
+ const node = require('lowdb/node');
9
+ const PQueue = require('p-queue');
13
10
  const cliProgress = require('cli-progress');
14
11
  const Options = require('cli-progress/lib/options.js');
15
12
  const _defaultFormatBar = require('cli-progress/lib/format-bar.js');
16
13
  const _defaultFormatTime = require('cli-progress/lib/format-time.js');
14
+ const dateFns = require('date-fns');
15
+ const fs = require('fs-extra');
16
+ const stripAnsi = require('strip-ansi');
17
+ const winston = require('winston');
17
18
 
18
19
  function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e.default : e; }
19
20
 
20
- const consola__default = /*#__PURE__*/_interopDefaultCompat(consola);
21
- const PQueue__default = /*#__PURE__*/_interopDefaultCompat(PQueue);
22
21
  const path__default = /*#__PURE__*/_interopDefaultCompat(path);
23
22
  const process__default = /*#__PURE__*/_interopDefaultCompat(process);
24
- const fs__default = /*#__PURE__*/_interopDefaultCompat(fs);
25
- const stripAnsi__default = /*#__PURE__*/_interopDefaultCompat(stripAnsi);
26
- const winston__default = /*#__PURE__*/_interopDefaultCompat(winston);
23
+ const consola__default = /*#__PURE__*/_interopDefaultCompat(consola);
24
+ const PQueue__default = /*#__PURE__*/_interopDefaultCompat(PQueue);
27
25
  const cliProgress__default = /*#__PURE__*/_interopDefaultCompat(cliProgress);
28
26
  const Options__default = /*#__PURE__*/_interopDefaultCompat(Options);
29
27
  const _defaultFormatBar__default = /*#__PURE__*/_interopDefaultCompat(_defaultFormatBar);
30
28
  const _defaultFormatTime__default = /*#__PURE__*/_interopDefaultCompat(_defaultFormatTime);
29
+ const fs__default = /*#__PURE__*/_interopDefaultCompat(fs);
30
+ const stripAnsi__default = /*#__PURE__*/_interopDefaultCompat(stripAnsi);
31
+ const winston__default = /*#__PURE__*/_interopDefaultCompat(winston);
31
32
 
32
33
  const siteUrlMap = /* @__PURE__ */ new Map();
33
34
  const PQueueMap = /* @__PURE__ */ new Map();
34
35
 
36
+ const formatter = (options, params, payload) => {
37
+ const { value, total } = params;
38
+ const formatTime = options.formatTime || _defaultFormatTime__default;
39
+ const formatBar = options.formatBar || _defaultFormatBar__default;
40
+ const bar = formatBar(params.progress, options);
41
+ const stopTime = params.stopTime || Date.now();
42
+ const elapsedTime = Math.round((stopTime - params.startTime) / 1e3);
43
+ const duration_formatted = formatTime(elapsedTime, options, 1);
44
+ consola__default.debug(duration_formatted);
45
+ if (!payload.type) {
46
+ options = Object.assign({}, options, cliProgress__default.Presets.rect);
47
+ Options__default.assignDerivedOptions(options);
48
+ const bar2 = formatBar(params.progress, options);
49
+ const { site } = payload;
50
+ const url = payload.url || "";
51
+ return `${bar2} ${value}/${total} | ${utils.colors.yellow(site || "MEODP Item")} | ${utils.colors.cyan(url || "URL")}`;
52
+ } else {
53
+ const { error_count, emoji, url, type } = payload;
54
+ const errorInfo = error_count ? `\u274C${error_count} ` : "";
55
+ return `${bar} \u2705${utils.colors.green(value)}/${total} ${errorInfo}| ${duration_formatted} | ${emoji} ${utils.colors.yellow(type)} | ${utils.colors.cyan(url)}`;
56
+ }
57
+ };
58
+ const multiBar = new cliProgress__default.MultiBar({
59
+ hideCursor: true,
60
+ // format: '{bar} {percentage}% | ✅{value}/{total} ❌{error_count} | {duration_formatted} | {name}',
61
+ format: formatter,
62
+ barsize: 20,
63
+ autopadding: true,
64
+ barCompleteChar: "\u2588",
65
+ barIncompleteChar: "\u2591",
66
+ // important! redraw everything to avoid "empty" completed bars
67
+ forceRedraw: true
68
+ }, cliProgress__default.Presets.rect);
69
+ const progressBarMap = /* @__PURE__ */ new Map();
70
+
35
71
  var __defProp$1 = Object.defineProperty;
36
72
  var __defNormalProp$1 = (obj, key, value) => key in obj ? __defProp$1(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
37
73
  var __publicField$1 = (obj, key, value) => {
@@ -164,40 +200,93 @@ const COLORFUL_SYMBOLS = {
164
200
  line: utils.colors.dim(RAW_SYMBOLS.line)
165
201
  };
166
202
 
167
- const formatter = (options, params, payload) => {
168
- const { value, total } = params;
169
- const formatTime = options.formatTime || _defaultFormatTime__default;
170
- const formatBar = options.formatBar || _defaultFormatBar__default;
171
- const bar = formatBar(params.progress, options);
172
- const stopTime = params.stopTime || Date.now();
173
- const elapsedTime = Math.round((stopTime - params.startTime) / 1e3);
174
- const duration_formatted = formatTime(elapsedTime, options, 1);
175
- consola__default.debug(duration_formatted);
176
- if (!payload.type) {
177
- options = Object.assign({}, options, cliProgress__default.Presets.rect);
178
- Options__default.assignDerivedOptions(options);
179
- const bar2 = formatBar(params.progress, options);
180
- const { site } = payload;
181
- const url = payload.url || "";
182
- return `${bar2} ${value}/${total} | ${utils.colors.yellow(site || "MEODP Item")} | ${utils.colors.cyan(url || "URL")}`;
183
- } else {
184
- const { error_count, emoji, url, type } = payload;
185
- const errorInfo = error_count ? `\u274C${error_count} ` : "";
186
- return `${bar} \u2705${utils.colors.green(value)}/${total} ${errorInfo}| ${duration_formatted} | ${emoji} ${utils.colors.yellow(type)} | ${utils.colors.cyan(url)}`;
187
- }
188
- };
189
- const multiBar = new cliProgress__default.MultiBar({
190
- hideCursor: true,
191
- // format: '{bar} {percentage}% | ✅{value}/{total} ❌{error_count} | {duration_formatted} | {name}',
192
- format: formatter,
193
- barsize: 20,
194
- autopadding: true,
195
- barCompleteChar: "\u2588",
196
- barIncompleteChar: "\u2591",
197
- // important! redraw everything to avoid "empty" completed bars
198
- forceRedraw: true
199
- }, cliProgress__default.Presets.rect);
200
- const progressBarMap = /* @__PURE__ */ new Map();
203
+ function createMEODPLogger() {
204
+ return {
205
+ _log: (...args) => {
206
+ const allLogPath = path__default.resolve(MEODP.logFolder, "all.log");
207
+ LocalLog.log(allLogPath, stripAnsi__default(args.join(" ")));
208
+ },
209
+ log: (...args) => {
210
+ const content = formatArgs(args);
211
+ consola__default.log(content);
212
+ MEODP.logger._log(content);
213
+ },
214
+ inner: (...args) => {
215
+ const content = formatArgs(args);
216
+ consola__default.log(COLORFUL_SYMBOLS.line, " ", content);
217
+ MEODP.logger._log(COLORFUL_SYMBOLS.line, " ", content);
218
+ },
219
+ start: (...args) => {
220
+ const content = formatArgs(args);
221
+ consola__default.start(content);
222
+ MEODP.logger._log(COLORFUL_SYMBOLS.start, content);
223
+ },
224
+ success: (...args) => {
225
+ const content = formatArgs(args);
226
+ consola__default.success(content);
227
+ MEODP.logger._log(COLORFUL_SYMBOLS.success, content);
228
+ },
229
+ info: (...args) => {
230
+ const content = formatArgs(args);
231
+ consola__default.info(content);
232
+ MEODP.logger._log(COLORFUL_SYMBOLS.info, content);
233
+ },
234
+ warn: (...args) => {
235
+ const content = formatArgs(args);
236
+ consola__default.warn(content);
237
+ MEODP.logger._log(COLORFUL_SYMBOLS.warn, content);
238
+ },
239
+ error: (...args) => {
240
+ const content = formatArgs(args);
241
+ consola__default.log(COLORFUL_SYMBOLS.error, content);
242
+ MEODP.logger._log(COLORFUL_SYMBOLS.error, content);
243
+ MEODP.wLogger.error(content);
244
+ }
245
+ };
246
+ }
247
+
248
+ function createWinstonLogger() {
249
+ const allLogPath = path__default.resolve(MEODP.logFolder, "all.log");
250
+ const errorLogPath = path__default.resolve(MEODP.logFolder, "error.log");
251
+ const customFormats = [
252
+ winston__default.format.timestamp({
253
+ format: "YYYY-MM-DD HH:mm:ss"
254
+ })
255
+ // winston.format.colorize(),
256
+ // winston.format.errors({ stack: true }),
257
+ // winston.format.json(),
258
+ // winston.format.simple(),
259
+ // winston.format.uncolorize(),
260
+ ];
261
+ customFormats.push(
262
+ winston__default.format.printf(({ level, message, timestamp, service }) => {
263
+ const content = [
264
+ timestamp,
265
+ `[${service}]`,
266
+ level.toUpperCase(),
267
+ message
268
+ ];
269
+ return content.join(" ");
270
+ })
271
+ );
272
+ const customTransports = [
273
+ new winston__default.transports.File({
274
+ filename: errorLogPath,
275
+ level: "error"
276
+ }),
277
+ new winston__default.transports.File({
278
+ filename: allLogPath
279
+ // level: 'info',
280
+ })
281
+ ];
282
+ return winston__default.createLogger({
283
+ level: "info",
284
+ format: winston__default.format.combine(...customFormats),
285
+ defaultMeta: { service: "MEODP" },
286
+ transports: customTransports,
287
+ exitOnError: false
288
+ });
289
+ }
201
290
 
202
291
  function formatArgs(args) {
203
292
  return args.join(" ");
@@ -208,7 +297,7 @@ const TIME_COLOR = {
208
297
  slow: utils.colors.red
209
298
  };
210
299
  function getFormattedDuration(duration) {
211
- const durationText = `${(Math.round(duration) / 1e3).toString()}s`;
300
+ const durationText = `${(duration / 1e3).toString()}s`;
212
301
  let type;
213
302
  if (duration < 500) {
214
303
  type = "fast";
@@ -225,9 +314,12 @@ function getFormattedDataFromResponse(res) {
225
314
  const statusInfo = statusText ? `${statusCode?.toString()} ${statusText}` : statusCode?.toString();
226
315
  const statusInfoText = statusCode >= 400 ? utils.colors.red(`[${statusInfo}]`) : utils.colors.green(`[${statusInfo}]`);
227
316
  const req = res.request();
228
- const duration = req.timing().responseEnd - req.timing().requestStart;
317
+ const duration = Math.round(req.timing().responseEnd - req.timing().requestStart);
229
318
  const durationText = `${COLORFUL_SYMBOLS.line} ${getFormattedDuration(duration)} ${COLORFUL_SYMBOLS.line}`;
230
319
  return {
320
+ /**
321
+ * 四舍五入 ms
322
+ */
231
323
  duration,
232
324
  durationText,
233
325
  statusCode,
@@ -375,92 +467,62 @@ function getMEODPUrlItemInfo(urlItem) {
375
467
  };
376
468
  }
377
469
 
378
- function createMEODPLogger() {
379
- return {
380
- _log: (...args) => {
381
- const allLogPath = path__default.resolve(MEODP.logFolder, "all.log");
382
- LocalLog.log(allLogPath, stripAnsi__default(args.join(" ")));
383
- },
384
- log: (...args) => {
385
- const content = formatArgs(args);
386
- consola__default.log(content);
387
- MEODP.logger._log(content);
388
- },
389
- inner: (...args) => {
390
- const content = formatArgs(args);
391
- consola__default.log(COLORFUL_SYMBOLS.line, " ", content);
392
- MEODP.logger._log(COLORFUL_SYMBOLS.line, " ", content);
393
- },
394
- start: (...args) => {
395
- const content = formatArgs(args);
396
- consola__default.start(content);
397
- MEODP.logger._log(COLORFUL_SYMBOLS.start, content);
398
- },
399
- success: (...args) => {
400
- const content = formatArgs(args);
401
- consola__default.success(content);
402
- MEODP.logger._log(COLORFUL_SYMBOLS.success, content);
403
- },
404
- info: (...args) => {
405
- const content = formatArgs(args);
406
- consola__default.info(content);
407
- MEODP.logger._log(COLORFUL_SYMBOLS.info, content);
408
- },
409
- warn: (...args) => {
410
- const content = formatArgs(args);
411
- consola__default.warn(content);
412
- MEODP.logger._log(COLORFUL_SYMBOLS.warn, content);
413
- },
414
- error: (...args) => {
415
- const content = formatArgs(args);
416
- consola__default.log(COLORFUL_SYMBOLS.error, content);
417
- MEODP.logger._log(COLORFUL_SYMBOLS.error, content);
418
- MEODP.wLogger.error(content);
470
+ const defaultMEODPConfig = {
471
+ urls: [],
472
+ concurrency: 10,
473
+ log: {
474
+ type: "raw",
475
+ file: true
476
+ }
477
+ };
478
+ function defineConfig(config) {
479
+ return config;
480
+ }
481
+ async function runByConfig(config) {
482
+ const { urls, debug } = config;
483
+ consola__default.level = debug ? 5 : 3;
484
+ const queue = new PQueue__default({
485
+ concurrency: config.concurrency
486
+ });
487
+ if (config.log?.type === "progress") {
488
+ for (const urlItem of urls) {
489
+ const { type, url, emoji } = getMEODPUrlItemInfo(urlItem);
490
+ const bar = multiBar.create(1, 0, {
491
+ emoji,
492
+ url,
493
+ type,
494
+ error_count: 0
495
+ });
496
+ progressBarMap.set(url, bar);
419
497
  }
420
- };
498
+ const curBar = multiBar.create(1, 0, {
499
+ name: "CURRENT"
500
+ });
501
+ progressBarMap.set("CURRENT", curBar);
502
+ }
503
+ const browser = await getBrowser();
504
+ for (const urlItem of urls) {
505
+ await queue.add(async () => {
506
+ await checkMEODPUrl(urlItem);
507
+ });
508
+ }
509
+ await queue.onIdle();
510
+ if (config.log?.type === "progress") {
511
+ multiBar.stop();
512
+ }
513
+ await browser.close();
514
+ consola__default.debug("browser closed");
515
+ return true;
421
516
  }
422
517
 
423
- function createWinstonLogger() {
424
- const allLogPath = path__default.resolve(MEODP.logFolder, "all.log");
425
- const errorLogPath = path__default.resolve(MEODP.logFolder, "error.log");
426
- const customFormats = [
427
- winston__default.format.timestamp({
428
- format: "YYYY-MM-DD HH:mm:ss"
429
- })
430
- // winston.format.colorize(),
431
- // winston.format.errors({ stack: true }),
432
- // winston.format.json(),
433
- // winston.format.simple(),
434
- // winston.format.uncolorize(),
435
- ];
436
- customFormats.push(
437
- winston__default.format.printf(({ level, message, timestamp, service }) => {
438
- const content = [
439
- timestamp,
440
- `[${service}]`,
441
- level.toUpperCase(),
442
- message
443
- ];
444
- return content.join(" ");
445
- })
446
- );
447
- const customTransports = [
448
- new winston__default.transports.File({
449
- filename: errorLogPath,
450
- level: "error"
451
- }),
452
- new winston__default.transports.File({
453
- filename: allLogPath
454
- // level: 'info',
455
- })
456
- ];
457
- return winston__default.createLogger({
458
- level: "info",
459
- format: winston__default.format.combine(...customFormats),
460
- defaultMeta: { service: "MEODP" },
461
- transports: customTransports,
462
- exitOnError: false
463
- });
518
+ const defaultData = {
519
+ config: defaultMEODPConfig,
520
+ sites: {}
521
+ };
522
+ async function createLowDB(rootDir = process__default.cwd()) {
523
+ const dbJsonPath = path__default.resolve(rootDir, "logs/meodp/db.json");
524
+ const db = await node.JSONFilePreset(dbJsonPath, defaultData);
525
+ return db;
464
526
  }
465
527
 
466
528
  var __defProp = Object.defineProperty;
@@ -485,6 +547,9 @@ const _MEODP = class _MEODP {
485
547
  await LocalLog.init();
486
548
  _MEODP.logFolder = LocalLog.logFolder || path__default.resolve(process__default.cwd(), "logs/meodp");
487
549
  _MEODP.wLogger = createWinstonLogger();
550
+ const db = await createLowDB();
551
+ _MEODP.db = db;
552
+ await db.update((data) => data.config = _MEODP.config);
488
553
  }
489
554
  /**
490
555
  * filter link by ignoreLinks
@@ -533,6 +598,7 @@ __publicField(_MEODP, "browser");
533
598
  __publicField(_MEODP, "logFolder");
534
599
  __publicField(_MEODP, "logger", createMEODPLogger());
535
600
  __publicField(_MEODP, "wLogger");
601
+ __publicField(_MEODP, "db");
536
602
  let MEODP = _MEODP;
537
603
 
538
604
  async function getBrowser() {
@@ -581,7 +647,6 @@ async function checkUrlNomoduleAssets(page) {
581
647
  if (!nomoduleUrl) {
582
648
  return;
583
649
  }
584
- const startTime = Date.now();
585
650
  const newPage = await context.newPage();
586
651
  const response = await newPage.goto(nomoduleUrl);
587
652
  if (!response) {
@@ -592,7 +657,8 @@ async function checkUrlNomoduleAssets(page) {
592
657
  statusInfo,
593
658
  linkText
594
659
  } = getFormattedDataFromResponse(response);
595
- const duration = Date.now() - startTime;
660
+ const req = response.request();
661
+ const duration = req.timing().responseEnd - req.timing().requestStart;
596
662
  const durationTxt = utils.colors.dim(`\u2502${utils.colors.blue(`${duration.toString().padStart(4, " ")}${utils.colors.white("ms")}`)} \u2502`);
597
663
  if (statusCode >= 400) {
598
664
  MEODP.logger.inner(" ", COLORFUL_SYMBOLS.error, utils.colors.red(statusInfo), durationTxt, linkText);
@@ -610,9 +676,31 @@ async function checkSiteUrl(url, options) {
610
676
  const browser = await getBrowser();
611
677
  const page = await browser.newPage();
612
678
  const bar = progressBarMap.get(options.urlItem.url);
613
- const startTime = Date.now();
614
679
  const isExternalLink = MEODP.isExternalLink(url, options.urlItem);
615
680
  const log = LocalLog.createLog(options.urlItem);
681
+ const siteData = MEODP.db.data.sites[options.urlItem.url];
682
+ const siteLinkItem = {
683
+ url,
684
+ statusCode: 0,
685
+ checkStatus: "pending",
686
+ success: 0,
687
+ failed: 0,
688
+ ignored: 0,
689
+ timeout: 0,
690
+ duration: 0,
691
+ total: 0,
692
+ requests: []
693
+ };
694
+ if (MEODP.isIgnoredLink(url)) {
695
+ siteUrlMap.set(url, {
696
+ statusCode: 0,
697
+ checkStatus: "ignored"
698
+ });
699
+ MEODP.logger.inner(`${utils.colors.dim(`[Ignored] ${utils.colors.underline(url)}`)}`);
700
+ siteData.ignored.push(url);
701
+ await MEODP.db.write();
702
+ return;
703
+ }
616
704
  if (!MEODP.config.checkExternalLinks && isExternalLink) {
617
705
  let res = null;
618
706
  try {
@@ -620,6 +708,7 @@ async function checkSiteUrl(url, options) {
620
708
  waitUntil: "load"
621
709
  });
622
710
  } catch (e) {
711
+ siteLinkItem.timeout = 1;
623
712
  MEODP.logger.error(e);
624
713
  }
625
714
  if (!res)
@@ -627,6 +716,7 @@ async function checkSiteUrl(url, options) {
627
716
  const {
628
717
  statusCode,
629
718
  statusInfoText,
719
+ duration,
630
720
  durationText
631
721
  } = getFormattedDataFromResponse(res);
632
722
  const checkStatus = statusCode < 400 ? "passed" : "failed";
@@ -639,28 +729,58 @@ async function checkSiteUrl(url, options) {
639
729
  if (statusCode >= 400) {
640
730
  MEODP.wLogger.error(` ${statusInfoText} ${url} ${durationText}`);
641
731
  }
732
+ siteLinkItem.statusCode = statusCode;
733
+ siteLinkItem.checkStatus = checkStatus;
734
+ siteLinkItem.duration = duration;
735
+ siteLinkItem.total = 1;
736
+ siteLinkItem.success = checkStatus === "passed" ? 1 : 0;
737
+ siteLinkItem.failed = checkStatus === "failed" ? 1 : 0;
738
+ siteLinkItem.requests = [];
739
+ siteData.links.push(siteLinkItem);
740
+ await MEODP.db.write();
642
741
  return;
643
742
  }
644
743
  const urlMap = registerPageEvents(page, options.urlItem);
645
- const { success, failed, total, ignored, timeout } = parseUrlMap(urlMap);
646
744
  try {
647
745
  const res = await page.goto(url, {
648
746
  waitUntil: "networkidle"
649
747
  });
650
748
  const statusCode = res?.status() || 0;
749
+ const statusText = res?.statusText();
750
+ const req = res?.request();
751
+ const duration = Math.round((req?.timing()?.responseEnd || 0) - (req?.timing()?.requestStart || 0));
651
752
  siteUrlMap.set(url, {
652
753
  response: res,
653
754
  statusCode,
654
755
  checkStatus: "goto"
655
756
  });
656
- const duration = (Date.now() - startTime) / 1e3;
657
- const statusText = res?.statusText();
757
+ const { success, failed, total, ignored, timeout } = parseUrlMap(urlMap);
758
+ siteLinkItem.title = await page.title();
759
+ siteLinkItem.statusCode = statusCode;
760
+ siteLinkItem.statusText = statusText;
761
+ siteLinkItem.checkStatus = "goto";
762
+ siteLinkItem.duration = duration;
763
+ siteLinkItem.total = total.count;
764
+ siteLinkItem.success = success.count;
765
+ siteLinkItem.failed = failed.count;
766
+ siteLinkItem.timeout = timeout.count;
767
+ siteLinkItem.ignored = ignored.count;
768
+ siteLinkItem.requests = Array.from(urlMap.values()).map((value) => {
769
+ const res2 = value.response;
770
+ return {
771
+ failed: value.failed,
772
+ ignored: value.ignored,
773
+ url: res2?.url(),
774
+ statusCode: res2?.status(),
775
+ statusText: res2?.statusText()
776
+ };
777
+ });
658
778
  const logInfo = [
659
779
  COLORFUL_SYMBOLS.line,
660
780
  " ",
661
781
  utils.colors.green(`[${statusCode}${statusText ? ` ${statusText}` : ""}]`),
662
782
  utils.colors.cyan(url),
663
- utils.colors.dim(`(in ${duration}s)`),
783
+ utils.colors.dim(`(in ${duration}ms)`),
664
784
  total.text,
665
785
  success.text,
666
786
  failed.text,
@@ -686,23 +806,25 @@ async function checkSiteUrl(url, options) {
686
806
  if (!info.response && !info.ignored) {
687
807
  MEODP.logger.log(COLORFUL_SYMBOLS.line, " ", COLORFUL_SYMBOLS.error, utils.colors.red("Timeout:"), utils.colors.underline(url2));
688
808
  MEODP.wLogger.error(` Timeout: ${url2}`);
809
+ siteLinkItem.checkStatus = "timeout";
689
810
  }
690
811
  }
691
812
  let noModuleAssetsPassed = true;
692
813
  if (options.checkNoModule) {
693
814
  noModuleAssetsPassed = await checkUrlNomoduleAssets(page);
694
815
  }
695
- const urlMapValues = Array.from(urlMap.values());
696
816
  const siteUrlItem = siteUrlMap.get(url);
697
817
  if (siteUrlItem) {
698
- if (urlMapValues.every((value) => value.response?.status() && value.response?.status() < 400) && noModuleAssetsPassed) {
818
+ const isAllRequestsPassed = siteLinkItem.requests.filter((value) => !value.ignored).every((value) => value.statusCode && value.statusCode < 400);
819
+ if (isAllRequestsPassed && noModuleAssetsPassed) {
699
820
  siteUrlItem.checkStatus = "passed";
700
- const successCount = Array.from(siteUrlMap.values()).filter((value) => value.checkStatus === "passed").length;
701
- bar?.update(successCount);
702
821
  } else {
703
822
  siteUrlItem.checkStatus = "failed";
704
823
  }
824
+ siteLinkItem.checkStatus = siteUrlItem.checkStatus;
705
825
  }
826
+ siteData.links.push(siteLinkItem);
827
+ await MEODP.db.write();
706
828
  if (!isExternalLink) {
707
829
  const queue = PQueueMap.get(url);
708
830
  if (queue) {
@@ -723,6 +845,7 @@ async function checkSiteUrl(url, options) {
723
845
  }
724
846
  }
725
847
  await page.close();
848
+ return siteLinkItem;
726
849
  }
727
850
  async function checkSite(props) {
728
851
  const { url } = props;
@@ -739,11 +862,25 @@ async function checkSite(props) {
739
862
  statusCode: 0,
740
863
  checkStatus: "pending"
741
864
  });
865
+ await MEODP.db.update((data) => {
866
+ data.sites[homeUrl] = {
867
+ name: props.name,
868
+ url: homeUrl,
869
+ checkStatus: "pending",
870
+ links: [],
871
+ ignored: []
872
+ };
873
+ });
742
874
  bar?.setTotal(siteUrlMap.size);
743
- await checkSiteUrl(homeUrl, {
875
+ const siteLinkItem = await checkSiteUrl(homeUrl, {
744
876
  urlItem: props,
745
877
  checkNoModule: true
746
878
  });
879
+ await MEODP.db.update((data) => {
880
+ const siteItem = data.sites[homeUrl];
881
+ siteItem.title = siteLinkItem?.title;
882
+ siteItem.checkStatus = siteItem.links.every((link) => link.checkStatus === "passed") ? "passed" : "failed";
883
+ });
747
884
  await queue.onIdle();
748
885
  await page.close();
749
886
  await context.close();
@@ -816,54 +953,6 @@ async function checkMEODPUrl(meodpUrl) {
816
953
  MEODP.logger.success(`\u{1F577}\uFE0F ${utils.colors.greenBright("[DONE]")} ${utils.colors.gray("in")} ${duration / 1e3}s.`);
817
954
  }
818
955
 
819
- const defaultMEODPConfig = {
820
- urls: [],
821
- concurrency: 10,
822
- log: {
823
- type: "raw",
824
- file: true
825
- }
826
- };
827
- function defineConfig(config) {
828
- return config;
829
- }
830
- async function runByConfig(config) {
831
- const { urls, debug } = config;
832
- consola__default.level = debug ? 5 : 3;
833
- const queue = new PQueue__default({
834
- concurrency: config.concurrency
835
- });
836
- if (config.log?.type === "progress") {
837
- for (const urlItem of urls) {
838
- const { type, url, emoji } = getMEODPUrlItemInfo(urlItem);
839
- const bar = multiBar.create(1, 0, {
840
- emoji,
841
- url,
842
- type,
843
- error_count: 0
844
- });
845
- progressBarMap.set(url, bar);
846
- }
847
- const curBar = multiBar.create(1, 0, {
848
- name: "CURRENT"
849
- });
850
- progressBarMap.set("CURRENT", curBar);
851
- }
852
- const browser = await getBrowser();
853
- for (const urlItem of urls) {
854
- await queue.add(async () => {
855
- await checkMEODPUrl(urlItem);
856
- });
857
- }
858
- await queue.onIdle();
859
- if (config.log?.type === "progress") {
860
- multiBar.stop();
861
- }
862
- await browser.close();
863
- consola__default.debug("browser closed");
864
- return true;
865
- }
866
-
867
956
  exports.LocalLog = LocalLog;
868
957
  exports.MEODP = MEODP;
869
958
  exports.PQueueMap = PQueueMap;
@@ -872,6 +961,7 @@ exports.checkMEODPUrl = checkMEODPUrl;
872
961
  exports.checkSite = checkSite;
873
962
  exports.checkSiteUrl = checkSiteUrl;
874
963
  exports.checkUrlNomoduleAssets = checkUrlNomoduleAssets;
964
+ exports.createLowDB = createLowDB;
875
965
  exports.createMEODPLogger = createMEODPLogger;
876
966
  exports.createWinstonLogger = createWinstonLogger;
877
967
  exports.defaultMEODPConfig = defaultMEODPConfig;
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "meodp",
3
3
  "type": "module",
4
- "version": "0.0.6",
4
+ "version": "0.0.7",
5
5
  "packageManager": "pnpm@9.12.3",
6
6
  "description": "Mystic Eyes of Death Perception. Find out dead links in your site.",
7
7
  "keywords": [
@@ -29,7 +29,9 @@
29
29
  "scripts": {
30
30
  "build": "unbuild",
31
31
  "dev": "unbuild --stub",
32
- "demo": "npm run example",
32
+ "dev:client": "pnpm -C client run dev",
33
+ "demo": "pnpm -C examples/app run dev",
34
+ "demo:export": "pnpm -C examples/app run export",
33
35
  "demo:app": "pnpm -C examples/app run dev",
34
36
  "example": "vite-node examples/check-site.ts",
35
37
  "test": "npm run test:e2e",
@@ -50,6 +52,8 @@
50
52
  "consola": "^3.2.3",
51
53
  "date-fns": "^4.1.0",
52
54
  "fs-extra": "^11.2.0",
55
+ "lowdb": "^7.0.1",
56
+ "markdown-table": "^3.0.4",
53
57
  "p-queue": "^8.0.1",
54
58
  "strip-ansi": "^7.1.0",
55
59
  "winston": "^3.17.0",