meodp 0.0.6 → 0.0.7

package/dist/shared/{meodp.9c5df07b.mjs → meodp.5489574b.mjs}

@@ -1,21 +1,57 @@
-import consola from 'consola';
-import PQueue from 'p-queue';
 import { colors } from 'consola/utils';
 import { chromium } from 'playwright';
 import path from 'node:path';
 import process from 'node:process';
-import { formatDate } from 'date-fns';
-import fs from 'fs-extra';
-import stripAnsi from 'strip-ansi';
-import winston from 'winston';
+import consola from 'consola';
+import { JSONFilePreset } from 'lowdb/node';
+import PQueue from 'p-queue';
 import cliProgress from 'cli-progress';
 import Options from 'cli-progress/lib/options.js';
 import _defaultFormatBar from 'cli-progress/lib/format-bar.js';
 import _defaultFormatTime from 'cli-progress/lib/format-time.js';
+import { formatDate } from 'date-fns';
+import fs from 'fs-extra';
+import stripAnsi from 'strip-ansi';
+import winston from 'winston';
 
 const siteUrlMap = /* @__PURE__ */ new Map();
 const PQueueMap = /* @__PURE__ */ new Map();
 
+const formatter = (options, params, payload) => {
+  const { value, total } = params;
+  const formatTime = options.formatTime || _defaultFormatTime;
+  const formatBar = options.formatBar || _defaultFormatBar;
+  const bar = formatBar(params.progress, options);
+  const stopTime = params.stopTime || Date.now();
+  const elapsedTime = Math.round((stopTime - params.startTime) / 1e3);
+  const duration_formatted = formatTime(elapsedTime, options, 1);
+  consola.debug(duration_formatted);
+  if (!payload.type) {
+    options = Object.assign({}, options, cliProgress.Presets.rect);
+    Options.assignDerivedOptions(options);
+    const bar2 = formatBar(params.progress, options);
+    const { site } = payload;
+    const url = payload.url || "";
+    return `${bar2} ${value}/${total} | ${colors.yellow(site || "MEODP Item")} | ${colors.cyan(url || "URL")}`;
+  } else {
+    const { error_count, emoji, url, type } = payload;
+    const errorInfo = error_count ? `\u274C${error_count} ` : "";
+    return `${bar} \u2705${colors.green(value)}/${total} ${errorInfo}| ${duration_formatted} | ${emoji} ${colors.yellow(type)} | ${colors.cyan(url)}`;
+  }
+};
+const multiBar = new cliProgress.MultiBar({
+  hideCursor: true,
+  // format: '{bar} {percentage}% | ✅{value}/{total} ❌{error_count} | {duration_formatted} | {name}',
+  format: formatter,
+  barsize: 20,
+  autopadding: true,
+  barCompleteChar: "\u2588",
+  barIncompleteChar: "\u2591",
+  // important! redraw everything to avoid "empty" completed bars
+  forceRedraw: true
+}, cliProgress.Presets.rect);
+const progressBarMap = /* @__PURE__ */ new Map();
+
 var __defProp$1 = Object.defineProperty;
 var __defNormalProp$1 = (obj, key, value) => key in obj ? __defProp$1(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
 var __publicField$1 = (obj, key, value) => {
@@ -148,40 +184,93 @@ const COLORFUL_SYMBOLS = {
   line: colors.dim(RAW_SYMBOLS.line)
 };
 
-const formatter = (options, params, payload) => {
-  const { value, total } = params;
-  const formatTime = options.formatTime || _defaultFormatTime;
-  const formatBar = options.formatBar || _defaultFormatBar;
-  const bar = formatBar(params.progress, options);
-  const stopTime = params.stopTime || Date.now();
-  const elapsedTime = Math.round((stopTime - params.startTime) / 1e3);
-  const duration_formatted = formatTime(elapsedTime, options, 1);
-  consola.debug(duration_formatted);
-  if (!payload.type) {
-    options = Object.assign({}, options, cliProgress.Presets.rect);
-    Options.assignDerivedOptions(options);
-    const bar2 = formatBar(params.progress, options);
-    const { site } = payload;
-    const url = payload.url || "";
-    return `${bar2} ${value}/${total} | ${colors.yellow(site || "MEODP Item")} | ${colors.cyan(url || "URL")}`;
-  } else {
-    const { error_count, emoji, url, type } = payload;
-    const errorInfo = error_count ? `\u274C${error_count} ` : "";
-    return `${bar} \u2705${colors.green(value)}/${total} ${errorInfo}| ${duration_formatted} | ${emoji} ${colors.yellow(type)} | ${colors.cyan(url)}`;
-  }
-};
-const multiBar = new cliProgress.MultiBar({
-  hideCursor: true,
-  // format: '{bar} {percentage}% | ✅{value}/{total} ❌{error_count} | {duration_formatted} | {name}',
-  format: formatter,
-  barsize: 20,
-  autopadding: true,
-  barCompleteChar: "\u2588",
-  barIncompleteChar: "\u2591",
-  // important! redraw everything to avoid "empty" completed bars
-  forceRedraw: true
-}, cliProgress.Presets.rect);
-const progressBarMap = /* @__PURE__ */ new Map();
+function createMEODPLogger() {
+  return {
+    _log: (...args) => {
+      const allLogPath = path.resolve(MEODP.logFolder, "all.log");
+      LocalLog.log(allLogPath, stripAnsi(args.join(" ")));
+    },
+    log: (...args) => {
+      const content = formatArgs(args);
+      consola.log(content);
+      MEODP.logger._log(content);
+    },
+    inner: (...args) => {
+      const content = formatArgs(args);
+      consola.log(COLORFUL_SYMBOLS.line, "  ", content);
+      MEODP.logger._log(COLORFUL_SYMBOLS.line, "  ", content);
+    },
+    start: (...args) => {
+      const content = formatArgs(args);
+      consola.start(content);
+      MEODP.logger._log(COLORFUL_SYMBOLS.start, content);
+    },
+    success: (...args) => {
+      const content = formatArgs(args);
+      consola.success(content);
+      MEODP.logger._log(COLORFUL_SYMBOLS.success, content);
+    },
+    info: (...args) => {
+      const content = formatArgs(args);
+      consola.info(content);
+      MEODP.logger._log(COLORFUL_SYMBOLS.info, content);
+    },
+    warn: (...args) => {
+      const content = formatArgs(args);
+      consola.warn(content);
+      MEODP.logger._log(COLORFUL_SYMBOLS.warn, content);
+    },
+    error: (...args) => {
+      const content = formatArgs(args);
+      consola.log(COLORFUL_SYMBOLS.error, content);
+      MEODP.logger._log(COLORFUL_SYMBOLS.error, content);
+      MEODP.wLogger.error(content);
+    }
+  };
+}
+
+function createWinstonLogger() {
+  const allLogPath = path.resolve(MEODP.logFolder, "all.log");
+  const errorLogPath = path.resolve(MEODP.logFolder, "error.log");
+  const customFormats = [
+    winston.format.timestamp({
+      format: "YYYY-MM-DD HH:mm:ss"
+    })
+    // winston.format.colorize(),
+    // winston.format.errors({ stack: true }),
+    // winston.format.json(),
+    // winston.format.simple(),
+    // winston.format.uncolorize(),
+  ];
+  customFormats.push(
+    winston.format.printf(({ level, message, timestamp, service }) => {
+      const content = [
+        timestamp,
+        `[${service}]`,
+        level.toUpperCase(),
+        message
+      ];
+      return content.join(" ");
+    })
+  );
+  const customTransports = [
+    new winston.transports.File({
+      filename: errorLogPath,
+      level: "error"
+    }),
+    new winston.transports.File({
+      filename: allLogPath
+      // level: 'info',
+    })
+  ];
+  return winston.createLogger({
+    level: "info",
+    format: winston.format.combine(...customFormats),
+    defaultMeta: { service: "MEODP" },
+    transports: customTransports,
+    exitOnError: false
+  });
+}
 
 function formatArgs(args) {
   return args.join(" ");
@@ -192,7 +281,7 @@ const TIME_COLOR = {
   slow: colors.red
 };
 function getFormattedDuration(duration) {
-  const durationText = `${(Math.round(duration) / 1e3).toString()}s`;
+  const durationText = `${(duration / 1e3).toString()}s`;
   let type;
   if (duration < 500) {
     type = "fast";
@@ -209,9 +298,12 @@ function getFormattedDataFromResponse(res) {
   const statusInfo = statusText ? `${statusCode?.toString()} ${statusText}` : statusCode?.toString();
   const statusInfoText = statusCode >= 400 ? colors.red(`[${statusInfo}]`) : colors.green(`[${statusInfo}]`);
   const req = res.request();
-  const duration = req.timing().responseEnd - req.timing().requestStart;
+  const duration = Math.round(req.timing().responseEnd - req.timing().requestStart);
   const durationText = `${COLORFUL_SYMBOLS.line} ${getFormattedDuration(duration)} ${COLORFUL_SYMBOLS.line}`;
   return {
+    /**
+     * 四舍五入 ms
+     */
     duration,
     durationText,
     statusCode,
@@ -359,92 +451,62 @@ function getMEODPUrlItemInfo(urlItem) {
   };
 }
 
-function createMEODPLogger() {
-  return {
-    _log: (...args) => {
-      const allLogPath = path.resolve(MEODP.logFolder, "all.log");
-      LocalLog.log(allLogPath, stripAnsi(args.join(" ")));
-    },
-    log: (...args) => {
-      const content = formatArgs(args);
-      consola.log(content);
-      MEODP.logger._log(content);
-    },
-    inner: (...args) => {
-      const content = formatArgs(args);
-      consola.log(COLORFUL_SYMBOLS.line, "  ", content);
-      MEODP.logger._log(COLORFUL_SYMBOLS.line, "  ", content);
-    },
-    start: (...args) => {
-      const content = formatArgs(args);
-      consola.start(content);
-      MEODP.logger._log(COLORFUL_SYMBOLS.start, content);
-    },
-    success: (...args) => {
-      const content = formatArgs(args);
-      consola.success(content);
-      MEODP.logger._log(COLORFUL_SYMBOLS.success, content);
-    },
-    info: (...args) => {
-      const content = formatArgs(args);
-      consola.info(content);
-      MEODP.logger._log(COLORFUL_SYMBOLS.info, content);
-    },
-    warn: (...args) => {
-      const content = formatArgs(args);
-      consola.warn(content);
-      MEODP.logger._log(COLORFUL_SYMBOLS.warn, content);
-    },
-    error: (...args) => {
-      const content = formatArgs(args);
-      consola.log(COLORFUL_SYMBOLS.error, content);
-      MEODP.logger._log(COLORFUL_SYMBOLS.error, content);
-      MEODP.wLogger.error(content);
+const defaultMEODPConfig = {
+  urls: [],
+  concurrency: 10,
+  log: {
+    type: "raw",
+    file: true
+  }
+};
+function defineConfig(config) {
+  return config;
+}
+async function runByConfig(config) {
+  const { urls, debug } = config;
+  consola.level = debug ? 5 : 3;
+  const queue = new PQueue({
+    concurrency: config.concurrency
+  });
+  if (config.log?.type === "progress") {
+    for (const urlItem of urls) {
+      const { type, url, emoji } = getMEODPUrlItemInfo(urlItem);
+      const bar = multiBar.create(1, 0, {
+        emoji,
+        url,
+        type,
+        error_count: 0
+      });
+      progressBarMap.set(url, bar);
     }
-  };
+    const curBar = multiBar.create(1, 0, {
+      name: "CURRENT"
+    });
+    progressBarMap.set("CURRENT", curBar);
+  }
+  const browser = await getBrowser();
+  for (const urlItem of urls) {
+    await queue.add(async () => {
+      await checkMEODPUrl(urlItem);
+    });
+  }
+  await queue.onIdle();
+  if (config.log?.type === "progress") {
+    multiBar.stop();
+  }
+  await browser.close();
+  consola.debug("browser closed");
+  return true;
 }
 
-function createWinstonLogger() {
-  const allLogPath = path.resolve(MEODP.logFolder, "all.log");
-  const errorLogPath = path.resolve(MEODP.logFolder, "error.log");
-  const customFormats = [
-    winston.format.timestamp({
-      format: "YYYY-MM-DD HH:mm:ss"
-    })
-    // winston.format.colorize(),
-    // winston.format.errors({ stack: true }),
-    // winston.format.json(),
-    // winston.format.simple(),
-    // winston.format.uncolorize(),
-  ];
-  customFormats.push(
-    winston.format.printf(({ level, message, timestamp, service }) => {
-      const content = [
-        timestamp,
-        `[${service}]`,
-        level.toUpperCase(),
-        message
-      ];
-      return content.join(" ");
-    })
-  );
-  const customTransports = [
-    new winston.transports.File({
-      filename: errorLogPath,
-      level: "error"
-    }),
-    new winston.transports.File({
-      filename: allLogPath
-      // level: 'info',
-    })
-  ];
-  return winston.createLogger({
-    level: "info",
-    format: winston.format.combine(...customFormats),
-    defaultMeta: { service: "MEODP" },
-    transports: customTransports,
-    exitOnError: false
-  });
+const defaultData = {
+  config: defaultMEODPConfig,
+  sites: {}
+};
+async function createLowDB(rootDir = process.cwd()) {
+  const dbJsonPath = path.resolve(rootDir, "logs/meodp/db.json");
+  const db = await JSONFilePreset(dbJsonPath, defaultData);
+  return db;
 }
 
 var __defProp = Object.defineProperty;
@@ -469,6 +531,9 @@ const _MEODP = class _MEODP {
     await LocalLog.init();
     _MEODP.logFolder = LocalLog.logFolder || path.resolve(process.cwd(), "logs/meodp");
     _MEODP.wLogger = createWinstonLogger();
+    const db = await createLowDB();
+    _MEODP.db = db;
+    await db.update((data) => data.config = _MEODP.config);
   }
   /**
    * filter link by ignoreLinks
@@ -517,6 +582,7 @@ __publicField(_MEODP, "browser");
 __publicField(_MEODP, "logFolder");
 __publicField(_MEODP, "logger", createMEODPLogger());
 __publicField(_MEODP, "wLogger");
+__publicField(_MEODP, "db");
 let MEODP = _MEODP;
 
 async function getBrowser() {
@@ -565,7 +631,6 @@ async function checkUrlNomoduleAssets(page) {
       if (!nomoduleUrl) {
         return;
       }
-      const startTime = Date.now();
       const newPage = await context.newPage();
       const response = await newPage.goto(nomoduleUrl);
       if (!response) {
@@ -576,7 +641,8 @@ async function checkUrlNomoduleAssets(page) {
         statusInfo,
         linkText
       } = getFormattedDataFromResponse(response);
-      const duration = Date.now() - startTime;
+      const req = response.request();
+      const duration = req.timing().responseEnd - req.timing().requestStart;
       const durationTxt = colors.dim(`\u2502${colors.blue(`${duration.toString().padStart(4, " ")}${colors.white("ms")}`)} \u2502`);
       if (statusCode >= 400) {
         MEODP.logger.inner(" ", COLORFUL_SYMBOLS.error, colors.red(statusInfo), durationTxt, linkText);
@@ -594,9 +660,31 @@ async function checkSiteUrl(url, options) {
   const browser = await getBrowser();
   const page = await browser.newPage();
   const bar = progressBarMap.get(options.urlItem.url);
-  const startTime = Date.now();
   const isExternalLink = MEODP.isExternalLink(url, options.urlItem);
   const log = LocalLog.createLog(options.urlItem);
+  const siteData = MEODP.db.data.sites[options.urlItem.url];
+  const siteLinkItem = {
+    url,
+    statusCode: 0,
+    checkStatus: "pending",
+    success: 0,
+    failed: 0,
+    ignored: 0,
+    timeout: 0,
+    duration: 0,
+    total: 0,
+    requests: []
+  };
+  if (MEODP.isIgnoredLink(url)) {
+    siteUrlMap.set(url, {
+      statusCode: 0,
+      checkStatus: "ignored"
+    });
+    MEODP.logger.inner(`${colors.dim(`[Ignored] ${colors.underline(url)}`)}`);
+    siteData.ignored.push(url);
+    await MEODP.db.write();
+    return;
+  }
   if (!MEODP.config.checkExternalLinks && isExternalLink) {
     let res = null;
     try {
@@ -604,6 +692,7 @@ async function checkSiteUrl(url, options) {
         waitUntil: "load"
       });
     } catch (e) {
+      siteLinkItem.timeout = 1;
       MEODP.logger.error(e);
     }
     if (!res)
@@ -611,6 +700,7 @@ async function checkSiteUrl(url, options) {
     const {
       statusCode,
       statusInfoText,
+      duration,
       durationText
     } = getFormattedDataFromResponse(res);
     const checkStatus = statusCode < 400 ? "passed" : "failed";
@@ -623,28 +713,58 @@ async function checkSiteUrl(url, options) {
     if (statusCode >= 400) {
       MEODP.wLogger.error(`  ${statusInfoText} ${url} ${durationText}`);
     }
+    siteLinkItem.statusCode = statusCode;
+    siteLinkItem.checkStatus = checkStatus;
+    siteLinkItem.duration = duration;
+    siteLinkItem.total = 1;
+    siteLinkItem.success = checkStatus === "passed" ? 1 : 0;
+    siteLinkItem.failed = checkStatus === "failed" ? 1 : 0;
+    siteLinkItem.requests = [];
+    siteData.links.push(siteLinkItem);
+    await MEODP.db.write();
     return;
   }
   const urlMap = registerPageEvents(page, options.urlItem);
-  const { success, failed, total, ignored, timeout } = parseUrlMap(urlMap);
   try {
     const res = await page.goto(url, {
       waitUntil: "networkidle"
     });
     const statusCode = res?.status() || 0;
+    const statusText = res?.statusText();
+    const req = res?.request();
+    const duration = Math.round((req?.timing()?.responseEnd || 0) - (req?.timing()?.requestStart || 0));
     siteUrlMap.set(url, {
       response: res,
       statusCode,
       checkStatus: "goto"
     });
-    const duration = (Date.now() - startTime) / 1e3;
-    const statusText = res?.statusText();
+    const { success, failed, total, ignored, timeout } = parseUrlMap(urlMap);
+    siteLinkItem.title = await page.title();
+    siteLinkItem.statusCode = statusCode;
+    siteLinkItem.statusText = statusText;
+    siteLinkItem.checkStatus = "goto";
+    siteLinkItem.duration = duration;
+    siteLinkItem.total = total.count;
+    siteLinkItem.success = success.count;
+    siteLinkItem.failed = failed.count;
+    siteLinkItem.timeout = timeout.count;
+    siteLinkItem.ignored = ignored.count;
+    siteLinkItem.requests = Array.from(urlMap.values()).map((value) => {
+      const res2 = value.response;
+      return {
+        failed: value.failed,
+        ignored: value.ignored,
+        url: res2?.url(),
+        statusCode: res2?.status(),
+        statusText: res2?.statusText()
+      };
+    });
     const logInfo = [
       COLORFUL_SYMBOLS.line,
       "  ",
       colors.green(`[${statusCode}${statusText ? ` ${statusText}` : ""}]`),
       colors.cyan(url),
-      colors.dim(`(in ${duration}s)`),
+      colors.dim(`(in ${duration}ms)`),
       total.text,
       success.text,
       failed.text,
@@ -670,23 +790,25 @@ async function checkSiteUrl(url, options) {
     if (!info.response && !info.ignored) {
       MEODP.logger.log(COLORFUL_SYMBOLS.line, "    ", COLORFUL_SYMBOLS.error, colors.red("Timeout:"), colors.underline(url2));
       MEODP.wLogger.error(`  Timeout: ${url2}`);
+      siteLinkItem.checkStatus = "timeout";
     }
   }
   let noModuleAssetsPassed = true;
   if (options.checkNoModule) {
     noModuleAssetsPassed = await checkUrlNomoduleAssets(page);
   }
-  const urlMapValues = Array.from(urlMap.values());
   const siteUrlItem = siteUrlMap.get(url);
   if (siteUrlItem) {
-    if (urlMapValues.every((value) => value.response?.status() && value.response?.status() < 400) && noModuleAssetsPassed) {
+    const isAllRequestsPassed = siteLinkItem.requests.filter((value) => !value.ignored).every((value) => value.statusCode && value.statusCode < 400);
+    if (isAllRequestsPassed && noModuleAssetsPassed) {
       siteUrlItem.checkStatus = "passed";
-      const successCount = Array.from(siteUrlMap.values()).filter((value) => value.checkStatus === "passed").length;
-      bar?.update(successCount);
     } else {
       siteUrlItem.checkStatus = "failed";
     }
+    siteLinkItem.checkStatus = siteUrlItem.checkStatus;
   }
+  siteData.links.push(siteLinkItem);
+  await MEODP.db.write();
   if (!isExternalLink) {
     const queue = PQueueMap.get(url);
     if (queue) {
@@ -707,6 +829,7 @@ async function checkSiteUrl(url, options) {
     }
   }
   await page.close();
+  return siteLinkItem;
 }
 async function checkSite(props) {
   const { url } = props;
@@ -723,11 +846,25 @@ async function checkSite(props) {
     statusCode: 0,
     checkStatus: "pending"
   });
+  await MEODP.db.update((data) => {
+    data.sites[homeUrl] = {
+      name: props.name,
+      url: homeUrl,
+      checkStatus: "pending",
+      links: [],
+      ignored: []
+    };
+  });
   bar?.setTotal(siteUrlMap.size);
-  await checkSiteUrl(homeUrl, {
+  const siteLinkItem = await checkSiteUrl(homeUrl, {
     urlItem: props,
     checkNoModule: true
   });
+  await MEODP.db.update((data) => {
+    const siteItem = data.sites[homeUrl];
+    siteItem.title = siteLinkItem?.title;
+    siteItem.checkStatus = siteItem.links.every((link) => link.checkStatus === "passed") ? "passed" : "failed";
+  });
   await queue.onIdle();
   await page.close();
   await context.close();
@@ -800,52 +937,4 @@ async function checkMEODPUrl(meodpUrl) {
   MEODP.logger.success(`\u{1F577}\uFE0F  ${colors.greenBright("[DONE]")} ${colors.gray("in")} ${duration / 1e3}s.`);
 }
 
-const defaultMEODPConfig = {
-  urls: [],
-  concurrency: 10,
-  log: {
-    type: "raw",
-    file: true
-  }
-};
-function defineConfig(config) {
-  return config;
-}
-async function runByConfig(config) {
-  const { urls, debug } = config;
-  consola.level = debug ? 5 : 3;
-  const queue = new PQueue({
-    concurrency: config.concurrency
-  });
-  if (config.log?.type === "progress") {
-    for (const urlItem of urls) {
-      const { type, url, emoji } = getMEODPUrlItemInfo(urlItem);
-      const bar = multiBar.create(1, 0, {
-        emoji,
-        url,
-        type,
-        error_count: 0
-      });
-      progressBarMap.set(url, bar);
-    }
-    const curBar = multiBar.create(1, 0, {
-      name: "CURRENT"
-    });
-    progressBarMap.set("CURRENT", curBar);
-  }
-  const browser = await getBrowser();
-  for (const urlItem of urls) {
-    await queue.add(async () => {
-      await checkMEODPUrl(urlItem);
-    });
-  }
-  await queue.onIdle();
-  if (config.log?.type === "progress") {
-    multiBar.stop();
-  }
-  await browser.close();
-  consola.debug("browser closed");
-  return true;
-}
-
-export { LocalLog as L, MEODP as M, PQueueMap as P, checkLink as a, getSiteLinks as b, checkMEODPUrl as c, checkUrlNomoduleAssets as d, checkSiteUrl as e, checkSite as f, getBrowser as g, defaultMEODPConfig as h, defineConfig as i, localLog as j, createMEODPLogger as k, logUrlItemInfo as l, createWinstonLogger as m, multiBar as n, registerPageEvents as o, progressBarMap as p, formatArgs as q, runByConfig as r, siteUrlMap as s, getFormattedDuration as t, getFormattedDataFromResponse as u, isLink as v, parseUrlMap as w, emojiMap as x, getMEODPUrlItemInfo as y };
+export { LocalLog as L, MEODP as M, PQueueMap as P, checkLink as a, getSiteLinks as b, checkMEODPUrl as c, checkUrlNomoduleAssets as d, checkSiteUrl as e, checkSite as f, getBrowser as g, defaultMEODPConfig as h, defineConfig as i, createLowDB as j, localLog as k, logUrlItemInfo as l, createMEODPLogger as m, createWinstonLogger as n, multiBar as o, progressBarMap as p, registerPageEvents as q, runByConfig as r, siteUrlMap as s, formatArgs as t, getFormattedDuration as u, getFormattedDataFromResponse as v, isLink as w, parseUrlMap as x, emojiMap as y, getMEODPUrlItemInfo as z };