meodp 0.0.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 云游君
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,41 @@
+# MEODP (Mystic Eyes of Death Perception)
+
+Written by Typescript, based on [Playwright](https://playwright.dev/).
+
+## Usage
+
+You can use it like a lib or a cli.
+
+Or like an application:
+
+```ts
+// Create a config file: `meodp.config.ts` in your project root.
+import { defineConfig } from 'meodp'
+
+export default defineConfig({
+
+})
+```
+
+## Ref
+
+- [lychee](https://lychee.cli.rs/)
+
+## Logs
+
+```bash
+logs/meodp
+```
+
+## FAQ
+
+### 与 lychee 的区别
+
+lychee 是一个使用 Rust 编写的快速检测链接的命令行。
+它的命令行应该能满足你使用命令行的大部分需求。
+
+但我希望能够通过脚本/配置自由地定制检测流程、日志，并记录相关内容，执行对应的函数。
+因此我们需要一个类似 SDK 的库，来实现类似的功能。
+同时基于 Typescript 可以获得更好的开发灵活性，而使用 Playwright 则可以模拟浏览器以检测页面中的资源加载。
+
+如果可能，它未来也许可以支持插件或预置配置。

package/bin/index.ts

@@ -0,0 +1,6 @@
+#!/usr/bin/env vite-node --script
+'use strict'
+
+import { run } from '../src/cli'
+
+run()

package/dist/cli/index.cjs

@@ -0,0 +1,76 @@
+'use strict';
+
+const process = require('node:process');
+const c12 = require('c12');
+const consola = require('consola');
+const utils = require('consola/utils');
+const yargs = require('yargs');
+const helpers = require('yargs/helpers');
+require('playwright');
+const config = require('../shared/meodp.6ae77875.cjs');
+require('node:path');
+require('strip-ansi');
+require('winston');
+require('winston/lib/winston/transports');
+require('p-queue');
+require('date-fns');
+require('fs-extra');
+require('cli-progress');
+require('cli-progress/lib/options');
+require('cli-progress/lib/format-bar');
+require('cli-progress/lib/format-time');
+
+function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e.default : e; }
+
+const process__default = /*#__PURE__*/_interopDefaultCompat(process);
+const consola__default = /*#__PURE__*/_interopDefaultCompat(consola);
+const yargs__default = /*#__PURE__*/_interopDefaultCompat(yargs);
+
+function commonOptions(args) {
+  return args.positional("root", {
+    default: ".",
+    type: "string",
+    describe: "root folder of your source files"
+  });
+}
+
+const __DEV__ = false;
+consola__default.level = 3;
+function debug(name, ...args) {
+  consola__default.debug(utils.colors.dim(name), ...args);
+}
+const cli = yargs__default(helpers.hideBin(process__default.argv)).scriptName("meodp").usage("$0 \u68C0\u6D4B\u94FE\u63A5").command(
+  "* [root]",
+  "Run MEODP to check links",
+  (args) => commonOptions(args),
+  async (argv) => {
+    debug("argv", argv);
+    const { root = process__default.cwd() } = argv;
+    const { config: config$1, configFile } = await c12.loadConfig({
+      cwd: root,
+      name: "meodp",
+      defaultConfig: config.defaultMEODPConfig
+    });
+    await config.MEODP.init(config$1);
+    debug("config", config$1);
+    config.MEODP.logger.info(`\u{1F6E0}\uFE0F  Config File: ${configFile}`);
+    config.MEODP.config = config$1;
+    config.MEODP.configFile = configFile || "";
+    config.MEODP.logger.start("\u{1F310} Start checking links...");
+    const startTime = Date.now();
+    await config.runByConfig(config$1);
+    const endTime = Date.now();
+    const duration = endTime - startTime;
+    console.log();
+    config.MEODP.logger.success(`All done! in ${utils.colors.yellow(duration / 1e3)}s.`);
+    process__default.exit(0);
+  }
+).alias("h", "help").alias("v", "version").showHelpOnFail(false).help();
+function run() {
+  cli.parse();
+}
+
+exports.__DEV__ = __DEV__;
+exports.cli = cli;
+exports.debug = debug;
+exports.run = run;

package/dist/cli/index.d.cts

@@ -0,0 +1,11 @@
+import * as yargs from 'yargs';
+
+declare const __DEV__ = false;
+/**
+ * @inner
+ */
+declare function debug(name: string, ...args: any[]): void;
+declare const cli: yargs.Argv<{}>;
+declare function run(): void;
+
+export { __DEV__, cli, debug, run };

package/dist/cli/index.d.mts

@@ -0,0 +1,11 @@
+import * as yargs from 'yargs';
+
+declare const __DEV__ = false;
+/**
+ * @inner
+ */
+declare function debug(name: string, ...args: any[]): void;
+declare const cli: yargs.Argv<{}>;
+declare function run(): void;
+
+export { __DEV__, cli, debug, run };

package/dist/cli/index.d.ts

@@ -0,0 +1,11 @@
+import * as yargs from 'yargs';
+
+declare const __DEV__ = false;
+/**
+ * @inner
+ */
+declare function debug(name: string, ...args: any[]): void;
+declare const cli: yargs.Argv<{}>;
+declare function run(): void;
+
+export { __DEV__, cli, debug, run };

package/dist/cli/index.mjs

@@ -0,0 +1,65 @@
+import process from 'node:process';
+import { loadConfig } from 'c12';
+import consola from 'consola';
+import { colors } from 'consola/utils';
+import yargs from 'yargs';
+import { hideBin } from 'yargs/helpers';
+import 'playwright';
+import { h as defaultMEODPConfig, M as MEODP, r as runByConfig } from '../shared/meodp.faa3c78c.mjs';
+import 'node:path';
+import 'strip-ansi';
+import 'winston';
+import 'winston/lib/winston/transports';
+import 'p-queue';
+import 'date-fns';
+import 'fs-extra';
+import 'cli-progress';
+import 'cli-progress/lib/options';
+import 'cli-progress/lib/format-bar';
+import 'cli-progress/lib/format-time';
+
+function commonOptions(args) {
+  return args.positional("root", {
+    default: ".",
+    type: "string",
+    describe: "root folder of your source files"
+  });
+}
+
+const __DEV__ = false;
+consola.level = 3;
+function debug(name, ...args) {
+  consola.debug(colors.dim(name), ...args);
+}
+const cli = yargs(hideBin(process.argv)).scriptName("meodp").usage("$0 \u68C0\u6D4B\u94FE\u63A5").command(
+  "* [root]",
+  "Run MEODP to check links",
+  (args) => commonOptions(args),
+  async (argv) => {
+    debug("argv", argv);
+    const { root = process.cwd() } = argv;
+    const { config, configFile } = await loadConfig({
+      cwd: root,
+      name: "meodp",
+      defaultConfig: defaultMEODPConfig
+    });
+    await MEODP.init(config);
+    debug("config", config);
+    MEODP.logger.info(`\u{1F6E0}\uFE0F  Config File: ${configFile}`);
+    MEODP.config = config;
+    MEODP.configFile = configFile || "";
+    MEODP.logger.start("\u{1F310} Start checking links...");
+    const startTime = Date.now();
+    await runByConfig(config);
+    const endTime = Date.now();
+    const duration = endTime - startTime;
+    console.log();
+    MEODP.logger.success(`All done! in ${colors.yellow(duration / 1e3)}s.`);
+    process.exit(0);
+  }
+).alias("h", "help").alias("v", "version").showHelpOnFail(false).help();
+function run() {
+  cli.parse();
+}
+
+export { __DEV__, cli, debug, run };

package/dist/index.cjs

@@ -0,0 +1,72 @@
+'use strict';
+
+const config = require('./shared/meodp.6ae77875.cjs');
+require('consola');
+require('p-queue');
+require('consola/utils');
+require('playwright');
+require('node:path');
+require('node:process');
+require('date-fns');
+require('fs-extra');
+require('strip-ansi');
+require('winston');
+require('winston/lib/winston/transports');
+require('cli-progress');
+require('cli-progress/lib/options');
+require('cli-progress/lib/format-bar');
+require('cli-progress/lib/format-time');
+
+async function checkSiteMap(urlItem) {
+  const { url } = urlItem;
+  const robotsUrl = new URL("/robots.txt", url).href;
+  const browser = await config.getBrowser();
+  const page = await browser.newPage();
+  await page.goto(robotsUrl);
+  const text = await page.textContent("pre");
+  const sitemap = text?.match(/Sitemap: (.*)/)?.[1];
+  let sitemapUrl = new URL(sitemap || "/sitemap.xml", url).href;
+  if (sitemap) {
+    sitemapUrl = sitemap.startsWith("http") ? sitemap : new URL(sitemap, url).href;
+  }
+  await page.goto(sitemapUrl);
+  const links = await page.$$eval("loc", (elements) => {
+    return elements.map((element) => element.textContent);
+  });
+  for (const link of links) {
+    if (link && !config.siteUrlMap.has(link)) {
+      config.siteUrlMap.set(link, {
+        statusCode: 0,
+        checkStatus: "pending"
+      });
+    }
+  }
+}
+
+exports.LocalLog = config.LocalLog;
+exports.PQueueMap = config.PQueueMap;
+exports.checkLink = config.checkLink;
+exports.checkMEODPUrl = config.checkMEODPUrl;
+exports.checkSite = config.checkSite;
+exports.checkSiteUrl = config.checkSiteUrl;
+exports.checkUrlNomoduleAssets = config.checkUrlNomoduleAssets;
+exports.createMEODPLogger = config.createMEODPLogger;
+exports.createWinstonLogger = config.createWinstonLogger;
+exports.defaultMEODPConfig = config.defaultMEODPConfig;
+exports.defineConfig = config.defineConfig;
+exports.emojiMap = config.emojiMap;
+exports.formatArgs = config.formatArgs;
+exports.getBrowser = config.getBrowser;
+exports.getFormattedDataFromResponse = config.getFormattedDataFromResponse;
+exports.getMEODPUrlItemInfo = config.getMEODPUrlItemInfo;
+exports.getSiteLinks = config.getSiteLinks;
+exports.isLink = config.isLink;
+exports.localLog = config.localLog;
+exports.logUrlItemInfo = config.logUrlItemInfo;
+exports.multiBar = config.multiBar;
+exports.parseUrlMap = config.parseUrlMap;
+exports.progressBarMap = config.progressBarMap;
+exports.registerPageEvents = config.registerPageEvents;
+exports.runByConfig = config.runByConfig;
+exports.siteUrlMap = config.siteUrlMap;
+exports.checkSiteMap = checkSiteMap;

package/dist/index.d.cts

@@ -0,0 +1,360 @@
+import * as PQueue from 'p-queue';
+import PQueue__default from 'p-queue';
+import * as playwright from 'playwright';
+import { Response, Request, Page } from 'playwright';
+import winston from 'winston';
+import cliProgress from 'cli-progress';
+
+/**
+ * 只检查内部链接
+ */
+declare const siteUrlMap: Map<string, {
+    response?: Response | null;
+    /**
+     * 状态码
+     */
+    statusCode: number;
+    /**
+     * 检查状态
+     */
+    checkStatus: "pending" | "passed" | "failed" | "goto";
+}>;
+/**
+ * site p-queue
+ */
+declare const PQueueMap: Map<string, PQueue__default<any, PQueue.QueueAddOptions>>;
+
+interface PageUrlEvent {
+    /**
+     * @desc 是否被忽略
+     */
+    ignored?: boolean;
+    request: Request;
+    response?: Response;
+    /**
+     * @desc 是否失败
+     */
+    failed?: boolean;
+}
+type UrlMap = Map<string, PageUrlEvent>;
+
+/**
+ * - site: 站点主页链接，将会检查对应的 sitemap
+ * - url: 单个页面链接
+ * - sitemap: sitemap.xml 链接，将会检查对应的 sitemap 中的链接
+ */
+type MEODPUrlType = 'site' | 'link' | 'sitemap';
+interface MEODPUrlProps {
+    /**
+     * @desc emoji
+     */
+    emoji?: string;
+    /**
+     * @desc 日志名称
+     */
+    name?: string;
+    /**
+     * - site: 站点主页链接，将会检查对应的 sitemap
+     * - url: 单个页面链接
+     * - sitemap: sitemap.xml 链接，将会检查对应的 sitemap 中的链接
+     * @default 'url'
+     */
+    type: MEODPUrlType;
+    /**
+     * url 为 html 时，将会检查页面上的所有资源
+     */
+    url: string;
+    /**
+     * @desc 站点 Headers
+     */
+    headers?: Record<string, string> & {
+        /**
+         * https://developer.mozilla.org/zh-CN/docs/Web/HTTP/Headers/Authorization
+         * 自定义 authorization 头
+         */
+        Authorization?: string;
+    };
+}
+type MEODPUrlItem = string | MEODPUrlProps;
+interface MEODPConfig {
+    /**
+     * @desc 是否清理缓存日志
+     * @default false
+     */
+    clean?: boolean;
+    /**
+     * @desc 是否开启 debug 模式
+     * @default false
+     */
+    debug?: boolean;
+    /**
+     * urls
+     * 可以是多种类型 @ref SiteUrlConfig
+     */
+    urls: MEODPUrlItem[];
+    /**
+     * 日志形式
+     */
+    log?: {
+        /**
+         * @default 'raw'
+         * @desc 日志类型
+         * - progress: 进度条
+         * - raw: 原始输出
+         */
+        type: 'progress' | 'raw';
+        /**
+         * @desc 是否输出到文件
+         * @default true
+         */
+        file?: boolean;
+    };
+    /**
+     * @desc 是否检查页面上的资源，包括图片、CSS、JS等
+     * @default false
+     */
+    checkAssets?: boolean;
+    /**
+     * @desc 是否检查外链中的资源
+     * @default true
+     */
+    checkExternalLinks?: boolean;
+    /**
+     * @todo
+     * @desc 忽略的链接
+     * @default []
+     * @example ignoreLinks: ['https://www.google.com']
+     * 被正则匹配成功或以字符串开头的链接将被忽略
+     */
+    ignoreLinks?: (string | RegExp)[];
+    /**
+     * 忽略的后缀名
+     * @example ignoreExtensions: ['.mp3', '.mp4']
+     */
+    ignoreExtensions?: string[];
+    /**
+     * @todo
+     * A List of accepted status codes for valid links
+     * @default [200, 204, 301, 302, 304]
+     */
+    acceptedStatusCodes?: number[];
+    /**
+     * @todo
+     * An extra function to determine if an url is valid
+     * @default (link: string) => true
+     */
+    isValidUrl?: (link: string) => boolean;
+    /**
+     * @todo
+     * @desc 是否检查 mail 链接
+     * @ref https://github.com/reacherhq/check-if-email-exists
+     * @default false
+     */
+    includeMail?: boolean;
+    /**
+     * @todo
+     * 重试次数
+     * @default 1
+     */
+    retryTimes?: number;
+    /**
+     * @todo
+     * 最大并发数
+     * @default 10
+     */
+    concurrency?: number;
+}
+
+/**
+ * check one link
+ */
+declare function checkLink(urlItem: MEODPUrlProps): Promise<void>;
+
+/**
+ * 获取站点外链
+ */
+declare function getSiteLinks(page: Page, options: CheckSiteUrlOptions): Promise<string[]>;
+/**
+ * <script nomodule=""></script> 不会被加载
+ * 手动匹配检测
+ */
+declare function checkUrlNomoduleAssets(page: Page): Promise<boolean>;
+interface CheckSiteUrlOptions {
+    urlItem: MEODPUrlProps;
+    /**
+     * 检查 nomodule 资源
+     */
+    checkNoModule?: boolean;
+}
+/**
+ * check url in site
+ * @param url
+ */
+declare function checkSiteUrl(url: string, options: CheckSiteUrlOptions): Promise<void>;
+/**
+ * 检查站点可访问性
+ * - 死链
+ * - 所有内嵌资源
+ */
+declare function checkSite(props: MEODPUrlProps): Promise<void>;
+
+/**
+ * 检查站点地图
+ */
+declare function checkSiteMap(urlItem: MEODPUrlProps): Promise<void>;
+
+/**
+ * @desc 检查 MEODP 链接
+ */
+declare function checkMEODPUrl(meodpUrl: MEODPUrlItem): Promise<void>;
+
+declare const defaultMEODPConfig: MEODPConfig;
+declare function defineConfig(config: MEODPConfig): MEODPConfig;
+/**
+ * 根据配置运行
+ */
+declare function runByConfig(config: MEODPConfig): Promise<boolean>;
+
+/**
+ * 获取浏览器实例
+ * - 单例模式
+ */
+declare function getBrowser(): Promise<playwright.Browser>;
+
+declare function logUrlItemInfo(urlItem: MEODPUrlItem): void;
+interface LocalLogOptions {
+    cwd?: string;
+    nameRule?: 'timestamp' | 'url';
+}
+/**
+ * @desc 本地日志 instance
+ */
+declare class LocalLog {
+    /**
+     * 日志根目录
+     */
+    static rootLogFolder: string;
+    /**
+     * rootLogFolder + yyyy-MM-dd-HH-mm-ss
+     */
+    static logFolder: string;
+    /**
+     * rootLogFolder + site
+     */
+    static siteLogFolder: string;
+    /**
+     * link.log
+     */
+    static linkLogFile: string;
+    /**
+     * 日志命名规则
+     * - timestamp: yyyy-MM-dd-HH-mm-ss 每次都会建立一个以时间戳命名的文件夹
+     * - url: url 在同一个文件夹下
+     */
+    static nameRule: 'timestamp' | 'url';
+    static options: LocalLogOptions;
+    constructor(options?: LocalLogOptions);
+    /**
+     * @desc init ensure dir
+     */
+    static init(options?: LocalLogOptions): Promise<void>;
+    /**
+     * @desc clean log
+     */
+    static clean(): Promise<void>;
+    static log(filePath: string, ...args: any[]): Promise<void>;
+    /**
+     * @desc append log
+     */
+    static SiteLog(urlItem: MEODPUrlProps, ...args: any[]): Promise<void>;
+    /**
+     * console link log
+     */
+    static linkLog(...args: any[]): Promise<void>;
+    static logByType(urlItem: MEODPUrlProps, ...args: any[]): Promise<void>;
+    /**
+     * @desc create log
+     */
+    static createLog(urlItem: MEODPUrlProps): (...args: any[]) => void;
+}
+declare const localLog: LocalLog;
+
+declare function createMEODPLogger(): {
+    _log: (...args: any[]) => void;
+    log: (...args: any[]) => void;
+    inner: (...args: any[]) => void;
+    start: (...args: any[]) => void;
+    success: (...args: any[]) => void;
+    info: (...args: any[]) => void;
+    warn: (...args: any[]) => void;
+    error: (...args: any[]) => void;
+};
+
+/**
+ * - winston
+ */
+declare function createWinstonLogger(): winston.Logger;
+
+declare const multiBar: cliProgress.MultiBar;
+/**
+ * Progress bar map
+ * created by multiBar.create
+ */
+declare const progressBarMap: Map<string, cliProgress.SingleBar>;
+
+declare function registerPageEvents(page: Page, urlItem: MEODPUrlProps): Map<string, PageUrlEvent>;
+
+declare function formatArgs(args: any[]): string;
+/**
+ * get formatted data from response
+ */
+declare function getFormattedDataFromResponse(res: Response): {
+    duration: number;
+    durationText: string;
+    statusCode: number;
+    statusText: string;
+    statusInfo: string;
+    linkText: string;
+};
+
+/**
+ * 是否为链接
+ */
+declare function isLink(url: string): boolean;
+
+interface UrlCategoryInfo {
+    count: number;
+    text: string;
+}
+/**
+ * get info from urlMap
+ */
+declare function parseUrlMap(urlMap: UrlMap): {
+    success: UrlCategoryInfo;
+    failed: UrlCategoryInfo;
+    /**
+     * Ignored links, controlled by `ignoreLinks` and `ignoreExtensions`
+     */
+    ignored: UrlCategoryInfo;
+    timeout: UrlCategoryInfo;
+    total: {
+        count: number;
+        text: string;
+    };
+};
+
+declare const emojiMap: {
+    site: string;
+    link: string;
+    sitemap: string;
+};
+/**
+ * get MEODPUrlItem info
+ */
+declare function getMEODPUrlItemInfo(urlItem: MEODPUrlItem): {
+    type: MEODPUrlType;
+    url: string;
+    emoji: string;
+};
+
+export { type CheckSiteUrlOptions, LocalLog, type LocalLogOptions, type MEODPConfig, type MEODPUrlItem, type MEODPUrlProps, type MEODPUrlType, PQueueMap, type PageUrlEvent, type UrlCategoryInfo, type UrlMap, checkLink, checkMEODPUrl, checkSite, checkSiteMap, checkSiteUrl, checkUrlNomoduleAssets, createMEODPLogger, createWinstonLogger, defaultMEODPConfig, defineConfig, emojiMap, formatArgs, getBrowser, getFormattedDataFromResponse, getMEODPUrlItemInfo, getSiteLinks, isLink, localLog, logUrlItemInfo, multiBar, parseUrlMap, progressBarMap, registerPageEvents, runByConfig, siteUrlMap };