@jayfong/x-utils 1.0.0-202407012223

package/package.json

@@ -0,0 +1,158 @@
+{
+  "name": "@jayfong/x-utils",
+  "description": "123456",
+  "version": "1.0.0-202407012223",
+  "type": "module",
+  "dependencies": {
+    "vtils": "^4.115.0",
+    "cli-table3": "^0.6.3",
+    "inquirer": "^9.2.13",
+    "clipboardy": "^4.0.0",
+    "execa": "^8.0.1",
+    "@jayfong/x-request": "^2.35.2",
+    "node-html-parser": "^6.1.12",
+    "csv": "^6.3.6",
+    "exit-hook": "^4.0.0",
+    "fs-extra": "^11.2.0",
+    "dotenv": "^16.4.5",
+    "docx": "^8.5.0",
+    "image-dimensions": "^2.3.0",
+    "@fastify/cors": "^9.0.1",
+    "fastify": "^4.26.0",
+    "nzh": "^1.0.12",
+    "pathe": "^1.1.2",
+    "pinyin-pro": "^3.19.4",
+    "puppeteer": "^22.3.0",
+    "consola": "^3.2.3",
+    "html-entities": "^2.5.2",
+    "file-type": "^19.0.0",
+    "sharp": "^0.32.5",
+    "yaml": "^2.3.4"
+  },
+  "exports": {
+    "./package.json": "./package.json",
+    "./ai": {
+      "require": "./_cjs/ai.js",
+      "import": "./ai.js",
+      "types": "./ai.d.ts"
+    },
+    "./cli": {
+      "require": "./_cjs/cli.js",
+      "import": "./cli.js",
+      "types": "./cli.d.ts"
+    },
+    "./clipboard": {
+      "require": "./_cjs/clipboard.js",
+      "import": "./clipboard.js",
+      "types": "./clipboard.d.ts"
+    },
+    "./cmd": {
+      "require": "./_cjs/cmd.js",
+      "import": "./cmd.js",
+      "types": "./cmd.d.ts"
+    },
+    "./crawler": {
+      "require": "./_cjs/crawler.js",
+      "import": "./crawler.js",
+      "types": "./crawler.d.ts"
+    },
+    "./csv": {
+      "require": "./_cjs/csv.js",
+      "import": "./csv.js",
+      "types": "./csv.d.ts"
+    },
+    "./data_set": {
+      "require": "./_cjs/data_set.js",
+      "import": "./data_set.js",
+      "types": "./data_set.d.ts"
+    },
+    "./env": {
+      "require": "./_cjs/env.js",
+      "import": "./env.js",
+      "types": "./env.d.ts"
+    },
+    "./fs": {
+      "require": "./_cjs/fs.js",
+      "import": "./fs.js",
+      "types": "./fs.d.ts"
+    },
+    "./html": {
+      "require": "./_cjs/html.js",
+      "import": "./html.js",
+      "types": "./html.d.ts"
+    },
+    "./http_server": {
+      "require": "./_cjs/http_server.js",
+      "import": "./http_server.js",
+      "types": "./http_server.d.ts"
+    },
+    "./json": {
+      "require": "./_cjs/json.js",
+      "import": "./json.js",
+      "types": "./json.d.ts"
+    },
+    "./ksb": {
+      "require": "./_cjs/ksb.js",
+      "import": "./ksb.js",
+      "types": "./ksb.d.ts"
+    },
+    "./number": {
+      "require": "./_cjs/number.js",
+      "import": "./number.js",
+      "types": "./number.d.ts"
+    },
+    "./ocr": {
+      "require": "./_cjs/ocr.js",
+      "import": "./ocr.js",
+      "types": "./ocr.d.ts"
+    },
+    "./path": {
+      "require": "./_cjs/path.js",
+      "import": "./path.js",
+      "types": "./path.d.ts"
+    },
+    "./pinyin": {
+      "require": "./_cjs/pinyin.js",
+      "import": "./pinyin.js",
+      "types": "./pinyin.d.ts"
+    },
+    "./puppeteer": {
+      "require": "./_cjs/puppeteer.js",
+      "import": "./puppeteer.js",
+      "types": "./puppeteer.d.ts"
+    },
+    "./runner": {
+      "require": "./_cjs/runner.js",
+      "import": "./runner.js",
+      "types": "./runner.d.ts"
+    },
+    "./task": {
+      "require": "./_cjs/task.js",
+      "import": "./task.js",
+      "types": "./task.d.ts"
+    },
+    "./text": {
+      "require": "./_cjs/text.js",
+      "import": "./text.js",
+      "types": "./text.d.ts"
+    },
+    "./upload": {
+      "require": "./_cjs/upload.js",
+      "import": "./upload.js",
+      "types": "./upload.d.ts"
+    },
+    "./yaml": {
+      "require": "./_cjs/yaml.js",
+      "import": "./yaml.js",
+      "types": "./yaml.d.ts"
+    },
+    "./unicode": {
+      "require": "./_cjs/unicode/unicode.js",
+      "import": "./unicode/unicode.js",
+      "types": "./unicode/unicode.d.ts"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/path.d.ts

@@ -0,0 +1,2 @@
+import path from 'pathe';
+export declare const XPath: typeof path;

package/path.js

@@ -0,0 +1,5 @@
+import path from "pathe";
+const XPath = path;
+export {
+  XPath
+};

package/pinyin.d.ts

@@ -0,0 +1,3 @@
+export declare class XPinyin {
+    static getTonelessPinyin(text: string): string[];
+}

package/pinyin.js

@@ -0,0 +1,12 @@
+import { pinyin } from "pinyin-pro";
+class XPinyin {
+  static getTonelessPinyin(text) {
+    return pinyin(text, {
+      type: "array",
+      toneType: "none"
+    });
+  }
+}
+export {
+  XPinyin
+};

package/puppeteer.d.ts

@@ -0,0 +1,38 @@
+/// <reference types="lodash" />
+import { Browser, ElementHandle, Page } from 'puppeteer';
+export type XPuppeteerBrowser = Browser;
+export type XPuppeteerPage = Page;
+export type XPuppeteerOpenBrowserOptions = {
+    headful?: boolean;
+    proxy?: string;
+    dataDir?: string;
+};
+export type XPuppeteerOpenPageOptions = {
+    match?: string | RegExp;
+    blockAssets?: boolean;
+    referer?: string;
+};
+export type XPuppeteerGetPageHtmlOptions = XPuppeteerOpenBrowserOptions & XPuppeteerOpenPageOptions & {
+    instanceId?: string | number;
+    useSystemBrowser?: boolean;
+    afterOpenPage?: (payload: {
+        browser: XPuppeteer;
+        page: XPuppeteerPage;
+    }) => any;
+};
+export declare class XPuppeteer {
+    browser: Browser;
+    connectSystemBrowser(): Promise<Browser>;
+    openBrowser(options?: XPuppeteerOpenBrowserOptions): Promise<Browser>;
+    openPage(urlOrHtml: string, options?: XPuppeteerOpenPageOptions): Promise<Page>;
+    input(page: Page, selector: string, textOrHtml: string, frameIndex?: number): Promise<void>;
+    selectAll(page: Page): Promise<void>;
+    paste(page: Page): Promise<void>;
+    queryByText<T extends Node = HTMLDivElement>(page: Page, text: string | RegExp | ((el: T) => boolean)): Promise<ElementHandle<T>>;
+    disconnect(): Promise<void>;
+    close(): Promise<void>;
+    static openBrowser(options?: XPuppeteerOpenBrowserOptions): Promise<XPuppeteer>;
+    static openPage(urlOrHtml: string, options?: XPuppeteerGetPageHtmlOptions): Promise<Page>;
+    static getPageHtmlXp: ((options: XPuppeteerGetPageHtmlOptions) => Promise<XPuppeteer>) & import("lodash").MemoizedFunction;
+    static getPageHtml(url: string, options?: XPuppeteerGetPageHtmlOptions): Promise<string>;
+}

package/puppeteer.js

@@ -0,0 +1,241 @@
+import { xr } from "@jayfong/x-request";
+import { asyncExitHook } from "exit-hook";
+import puppeteer from "puppeteer";
+import { isFunction, isRegExp, memoize } from "vtils";
+import { XClipboard } from "./clipboard.js";
+class XPuppeteer {
+  browser;
+  async connectSystemBrowser() {
+    const {
+      data: { webSocketDebuggerUrl }
+    } = await xr.getJson({
+      url: "http://localhost:9222/json/version"
+    });
+    const browser = await puppeteer.connect({
+      browserWSEndpoint: webSocketDebuggerUrl,
+      // 都设为 0 以保持原有的宽高
+      defaultViewport: {
+        width: 0,
+        height: 0
+      }
+    });
+    this.browser = browser;
+    asyncExitHook(() => this.disconnect(), {
+      wait: 1e3
+    });
+    return browser;
+  }
+  async openBrowser(options = {}) {
+    const browser = await puppeteer.launch({
+      headless: options.headful ? false : true,
+      defaultViewport: {
+        width: 1440,
+        height: 900
+      },
+      userDataDir: options.dataDir,
+      args: options.proxy ? [`--proxy-server=${options.proxy}`] : void 0
+    });
+    this.browser = browser;
+    asyncExitHook(() => this.close(), {
+      wait: 1e3
+    });
+    return browser;
+  }
+  async openPage(urlOrHtml, options = {}) {
+    const isUrl = /^https?:\/\//i.test(urlOrHtml);
+    const pages = await this.browser.pages();
+    if (isUrl) {
+      const inputUrl = new URL(urlOrHtml);
+      const foundPage = pages.find((item) => {
+        const pageUrl = item.url();
+        const url = new URL(pageUrl);
+        return inputUrl.origin === url.origin && (pageUrl === urlOrHtml || !!options.match && (typeof options.match === "string" ? `${url.pathname}${url.search}`.includes(options.match) : options.match.test(`${url.pathname}${url.search}`)));
+      });
+      if (foundPage) {
+        return foundPage;
+      }
+    }
+    const page2 = await this.browser.newPage();
+    await page2.setUserAgent(
+      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
+    );
+    if (options.blockAssets) {
+      await page2.setRequestInterception(true);
+      page2.on("request", (req) => {
+        const resourceType = req.resourceType();
+        if (resourceType === "stylesheet" || resourceType === "font" || resourceType === "image") {
+          req.abort();
+        } else {
+          req.continue();
+        }
+      });
+    }
+    if (isUrl) {
+      await page2.goto(urlOrHtml, {
+        referer: options.referer || urlOrHtml
+      });
+    } else {
+      await page2.setContent(urlOrHtml);
+    }
+    return page2;
+  }
+  async input(page2, selector, textOrHtml, frameIndex) {
+    const mainPage = page2;
+    const keyboard = page2.keyboard;
+    if (frameIndex != null) {
+      page2 = page2.frames()[frameIndex];
+    }
+    const el2 = await page2.$(selector);
+    const isText = await el2.evaluate((el3) => {
+      const isInput = el3.tagName === "INPUT" || el3.tagName === "TEXTAREA";
+      const isText2 = isInput || el3.tagName === "SPAN";
+      return isText2;
+    });
+    if (isText) {
+      await XClipboard.writeText(textOrHtml);
+    } else {
+      await XClipboard.writeHtml(textOrHtml);
+    }
+    await el2.click();
+    await this.selectAll(page2);
+    await keyboard.press("Backspace");
+    await this.paste(mainPage);
+  }
+  async selectAll(page2) {
+    await page2.evaluate(() => {
+      const isMac = /Mac|iPod|iPhone|iPad/.test(window.navigator.platform);
+      document.activeElement.dispatchEvent(
+        new KeyboardEvent("keydown", {
+          bubbles: true,
+          cancelable: true,
+          key: isMac ? "Meta" : "Control",
+          code: isMac ? "MetaLeft" : "ControlLeft",
+          location: window.KeyboardEvent.DOM_KEY_LOCATION_LEFT,
+          // @ts-ignore
+          getModifierState: (keyArg) => keyArg === (isMac ? "Meta" : "Control"),
+          ctrlKey: !isMac,
+          metaKey: isMac,
+          charCode: 0,
+          keyCode: isMac ? 93 : 17,
+          which: isMac ? 93 : 17
+        })
+      );
+      const preventableEvent = new KeyboardEvent("keydown", {
+        bubbles: true,
+        cancelable: true,
+        key: "a",
+        code: "KeyA",
+        location: window.KeyboardEvent.DOM_KEY_LOCATION_STANDARD,
+        // @ts-ignore
+        getModifierState: (keyArg) => keyArg === (isMac ? "Meta" : "Control"),
+        ctrlKey: !isMac,
+        metaKey: isMac,
+        charCode: 0,
+        keyCode: 65,
+        which: 65
+      });
+      const wasPrevented = !document.activeElement.dispatchEvent(preventableEvent) || preventableEvent.defaultPrevented;
+      if (!wasPrevented) {
+        document.execCommand("selectall", false, null);
+      }
+      document.activeElement.dispatchEvent(
+        new KeyboardEvent("keyup", {
+          bubbles: true,
+          cancelable: true,
+          key: isMac ? "Meta" : "Control",
+          code: isMac ? "MetaLeft" : "ControlLeft",
+          location: window.KeyboardEvent.DOM_KEY_LOCATION_LEFT,
+          // @ts-ignore
+          getModifierState: () => false,
+          charCode: 0,
+          keyCode: isMac ? 93 : 17,
+          which: isMac ? 93 : 17
+        })
+      );
+    });
+  }
+  async paste(page2) {
+    await page2.keyboard.down("Shift");
+    await page2.keyboard.press("Insert");
+    await page2.keyboard.up("Shift");
+  }
+  async queryByText(page, text) {
+    const el = await page.evaluateHandle(
+      (text, isRegExp, isFunction) => {
+        const textRe = isRegExp ? (() => {
+          const m = text.match(/^\/(.+)\/([^/]*)$/);
+          return new RegExp(m[1], m[2] || void 0);
+        })() : void 0;
+        const textFn = isFunction ? eval(`(() => {return ${text}})()`) : void 0;
+        const walker = document.createTreeWalker(
+          document.body,
+          NodeFilter.SHOW_TEXT,
+          {
+            acceptNode: (node) => {
+              return (isRegExp ? textRe.test(node.textContent) : isFunction ? textFn(node) : node.textContent.includes(text)) ? NodeFilter.FILTER_ACCEPT : NodeFilter.FILTER_REJECT;
+            }
+          }
+        );
+        walker.nextNode();
+        return walker.currentNode.parentElement;
+      },
+      String(text),
+      isRegExp(text),
+      isFunction(text)
+    );
+    return el;
+  }
+  async disconnect() {
+    await this.browser.disconnect();
+  }
+  async close() {
+    await this.browser.close();
+  }
+  static async openBrowser(options = {}) {
+    const xp = new XPuppeteer();
+    await xp.openBrowser(options);
+    return xp;
+  }
+  static async openPage(urlOrHtml, options = {}) {
+    const xp = new XPuppeteer();
+    await xp.openBrowser(options);
+    const page2 = await xp.openPage(urlOrHtml, options);
+    page2.close = () => xp.close();
+    return page2;
+  }
+  static getPageHtmlXp = memoize(
+    async (options) => {
+      const xp = new XPuppeteer();
+      await (options.useSystemBrowser ? xp.connectSystemBrowser() : xp.openBrowser(options));
+      return xp;
+    },
+    (options) => options.instanceId
+  );
+  static async getPageHtml(url, options = {}) {
+    const hasInstanceId = !!options.instanceId;
+    options.instanceId ||= Date.now();
+    const xp = await XPuppeteer.getPageHtmlXp(options);
+    try {
+      const page2 = await xp.openPage(url, options);
+      if (options.afterOpenPage) {
+        await options.afterOpenPage({
+          browser: xp,
+          page: page2
+        });
+      }
+      const html = await page2.content();
+      return html;
+    } finally {
+      if (!hasInstanceId) {
+        if (options.useSystemBrowser) {
+          await xp.disconnect();
+        } else {
+          await xp.close();
+        }
+      }
+    }
+  }
+}
+export {
+  XPuppeteer
+};

package/runner.d.ts

@@ -0,0 +1,13 @@
+import { DistinctQuestion } from 'inquirer';
+import { MsValue } from 'vtils';
+export type XRunnerOptions<TConfig extends Record<any, any>> = {
+    configFile: string;
+    delay: MsValue | ((retryCount: number) => MsValue);
+    questions: Array<DistinctQuestion<TConfig>>;
+    run: (config: TConfig) => any;
+};
+export declare class XRunner<TConfig extends Record<any, any>> {
+    private options;
+    constructor(options: XRunnerOptions<TConfig>);
+    run(): Promise<void>;
+}

package/runner.js

@@ -0,0 +1,58 @@
+import inquirer from "inquirer";
+import { ms, wait } from "vtils";
+import { XFs } from "./fs.js";
+class XRunner {
+  constructor(options) {
+    this.options = options;
+  }
+  async run() {
+    await XFs.ensureFile(this.options.configFile);
+    const savedConfig = await XFs.readJson(this.options.configFile, { throws: false }) || {};
+    const config = await inquirer.prompt(
+      this.options.questions.map((item) => {
+        return {
+          ...item,
+          default: savedConfig[item.name] ?? item.default
+        };
+      })
+    );
+    await XFs.writeJson(this.options.configFile, config, { spaces: 2 });
+    let retryCount = 0;
+    while (true) {
+      try {
+        await this.options.run(config);
+        retryCount = 0;
+      } catch (err) {
+        const delay = typeof this.options.delay === "function" ? this.options.delay(retryCount) : this.options.delay;
+        console.log(err);
+        console.log(`\u6267\u884C\u62A5\u9519, \u7B49\u5F85${delay}`);
+        await wait(ms(delay));
+        retryCount++;
+      }
+    }
+  }
+}
+new XRunner({
+  configFile: "./temp/test.json",
+  delay: "1s",
+  questions: [
+    {
+      name: "perCount",
+      message: "\u6BCF\u8F6E\u5904\u7406\u6761\u6570",
+      type: "number",
+      default: 500
+    },
+    {
+      name: "concurrencyCount",
+      message: "\u6BCF\u8F6E\u5E76\u884C\u5904\u7406\u6761\u6570",
+      type: "number",
+      default: 1
+    }
+  ],
+  run: (config) => {
+    console.log(config);
+  }
+}).run();
+export {
+  XRunner
+};

package/task.d.ts

@@ -0,0 +1,6 @@
+export declare abstract class XTask {
+    log: import("consola/core").ConsolaInstance;
+    abstract start(): Promise<void>;
+    dispose(): Promise<void>;
+    run(): Promise<void>;
+}

package/task.js

@@ -0,0 +1,21 @@
+import { consola as log } from "consola";
+import { asyncExitHook, gracefulExit } from "exit-hook";
+class XTask {
+  log = log;
+  async dispose() {
+  }
+  run() {
+    asyncExitHook(() => this.dispose(), {
+      wait: 3e3
+    });
+    return this.start().then(() => {
+      gracefulExit(0);
+    }).catch((err) => {
+      log.error(err);
+      gracefulExit(1);
+    });
+  }
+}
+export {
+  XTask
+};

package/text.d.ts

@@ -0,0 +1,63 @@
+export declare class XText {
+    private text;
+    /**
+     * 将字符串分割为字符数组。
+     *
+     * @see https://stackoverflow.com/a/71619350
+     */
+    static split(text: string): string[];
+    /**
+     * 纯粹为了书写 HTML 有提示。
+     */
+    static html(literals: any, ...interpolations: any): string;
+    /**
+     * 切割字符串。
+     */
+    static cut(text: string, start: string, end: string): string;
+    /**
+     * 移除全部空白字符。
+     */
+    static stripWhitespace(text: string): string;
+    /**
+     * 移除首尾空白字符。
+     */
+    static stripMarginWhitespace(text: string): string;
+    /**
+     * HTML 实体编码。
+     */
+    static encodeHtmlEntity(text: string): string;
+    /**
+     * HTML 实体解码。
+     */
+    static decodeHtmlEntity(text: string): string;
+    /**
+     * 移除 HTML 代码。
+     */
+    static stripHtml(text: string): string;
+    constructor(text: string);
+    /**
+     * 切割字符串。
+     */
+    cut(start: string, end: string): this;
+    /**
+     * 移除全部空白字符。
+     */
+    stripWhitespace(): this;
+    /**
+     * 移除首尾空白字符。
+     */
+    stripMarginWhitespace(): this;
+    /**
+     * HTML 实体编码。
+     */
+    encodeHtmlEntity(): this;
+    /**
+     * HTML 实体解码。
+     */
+    decodeHtmlEntity(): this;
+    /**
+     * 移除 HTML 代码。
+     */
+    stripHtml(): this;
+    toString(): string;
+}