@jayfong/x-utils 1.0.0-202407012223

package/cli.js

@@ -0,0 +1,21 @@
+import Table from "cli-table3";
+import inquirer from "inquirer";
+class XCli {
+  static async prompt(options) {
+    const { x } = await inquirer.prompt({
+      name: "x",
+      message: options.message,
+      type: options.type,
+      default: options.default
+    });
+    return x;
+  }
+  static table(data) {
+    const table = new Table();
+    table.push(...data);
+    return table.toString();
+  }
+}
+export {
+  XCli
+};

package/clipboard.d.ts

@@ -0,0 +1,6 @@
+export declare class XClipboard {
+    static readText(): Promise<string>;
+    static readHtml(): Promise<string>;
+    static writeText(text: string): Promise<void>;
+    static writeHtml(html: string): Promise<void>;
+}

package/clipboard.js

@@ -0,0 +1,35 @@
+import clipboardy from "clipboardy";
+import { XCmd } from "./cmd.js";
+class XClipboard {
+  static async readText() {
+    return clipboardy.read();
+  }
+  static async readHtml() {
+    const hex = await XCmd.exec(
+      // 读取剪切板为 HTML
+      // https://stackoverflow.com/questions/2545289/getting-rtf-data-out-of-mac-os-x-pasteboard-clipboard
+      // https://stackoverflow.com/a/17217886
+      `osascript -e "the clipboard as \xABclass HTML\xBB"`
+    );
+    const html = Buffer.from(
+      hex.substring(10, hex.length - 1),
+      "hex"
+    ).toString();
+    return html;
+  }
+  static async writeText(text) {
+    await clipboardy.write(text);
+  }
+  static async writeHtml(html) {
+    const hex = Buffer.from(html).toString("hex");
+    await XCmd.exec(
+      // 设置剪切板为 HTML
+      // https://stackoverflow.com/questions/11085654/apple-script-how-can-i-copy-html-content-to-the-clipboard
+      // https://stackoverflow.com/a/11089226
+      `osascript -e "set the clipboard to \xABdata HTML${hex}\xBB"`
+    );
+  }
+}
+export {
+  XClipboard
+};

package/cmd.d.ts

@@ -0,0 +1,5 @@
+import { Options } from 'execa';
+export declare class XCmd {
+    private static execDivider;
+    static exec(command: string, options?: Options): Promise<string>;
+}

package/cmd.js

@@ -0,0 +1,22 @@
+import { execa } from "execa";
+class XCmd {
+  static execDivider = "clrue91c60000e6rl7w03b40p";
+  static async exec(command, options) {
+    const { stdout } = await execa(
+      "zsh",
+      [
+        "-c",
+        `
+          echo ${XCmd.execDivider}
+          ${command}
+          echo ${XCmd.execDivider}
+        `
+      ],
+      options
+    );
+    return stdout.split(XCmd.execDivider)[1].trim();
+  }
+}
+export {
+  XCmd
+};

package/crawler.d.ts

@@ -0,0 +1,162 @@
+import { HTMLElement } from 'node-html-parser';
+import { EventBus } from 'vtils';
+import { AsyncOrSync, OneOrMore } from 'vtils/types';
+import { VaeObjectSchemaShapeOf, v } from 'vtils/vae';
+import { XPuppeteerGetPageHtmlOptions } from './puppeteer';
+export type XCrawlerData = Record<any, any>;
+export type XCrawlerDataFieldSelectSelector = string;
+export type XCrawlerDataFieldSelectFilter = [
+    string,
+    string | RegExp | ((el: HTMLElement) => boolean)
+];
+export type XCrawlerDataFieldSelectFunction = (el: HTMLElement) => AsyncOrSync<string | HTMLElement | undefined>;
+export type XCrawlerDataFieldSelect = XCrawlerDataFieldSelectSelector | XCrawlerDataFieldSelectFilter | XCrawlerDataFieldSelectFunction;
+export type XCrawlerDataFieldExtractExtract = 'getPlainText' | 'getHTML' | 'getFirstImageUrl' | 'getFirstLinkUrl';
+export type XCrawlerDataFieldExtract<V> = {
+    /**
+     * 选择元素
+     *
+     * - 可以是一个选择器字符串；
+     * - 可以是一个筛选器数组，[选择器, 筛选条件]；
+     * - 也可以是一个返回元素的函数；
+     * - 也可以是一个返回文本的函数，若此，将跳过 ocr 和 action。
+     */
+    select?: XCrawlerDataFieldSelect;
+    /**
+     * 是否识别选中内容中的图片为文本并替换
+     *
+     * @default false
+     */
+    ocr?: boolean;
+    /**
+     * 对选中元素导出结果文本
+     *
+     * @default 'getPlainText'
+     */
+    extract?: XCrawlerDataFieldExtractExtract;
+    /**
+     * 若有需要，可对结果文本进行转换
+     *
+     * @param text 结果文本
+     */
+    transform?: (text: string) => AsyncOrSync<V>;
+};
+export type XCrawlerDataExtract<T extends XCrawlerData> = {
+    [K in keyof T]?: XCrawlerDataFieldSelect | XCrawlerDataFieldExtract<T[K]>;
+};
+export type XCrawlerEngine = 'headless' | 'headful' | 'request';
+export type XCrawlerContentType = 'html' | 'json';
+export type XCrawlerNext<T extends XCrawlerData> = {
+    /**
+     * ID，可给其他 next 跳转以实现逻辑复用
+     */
+    id?: string;
+    /**
+     * 过滤
+     */
+    filter?: (data: T) => AsyncOrSync<boolean>;
+    /**
+     * 内容类型
+     */
+    contentType?: XCrawlerContentType;
+    /**
+     * 通过元素
+     */
+    element?: string | ((el: HTMLElement, extra: {
+        index: number;
+        root: HTMLElement;
+    }) => AsyncOrSync<OneOrMore<HTMLElement>>);
+    /**
+     * 通过字段(json)
+     */
+    field?: string | ((json: any) => AsyncOrSync<any>);
+    /**
+     * 通过网址
+     */
+    url?: keyof T | ((payload: {
+        data: T;
+        el: HTMLElement;
+    }) => AsyncOrSync<OneOrMore<string>>);
+    /**
+     * 下一页网址（有分页时用）
+     *
+     * 返回空字符串或跟当前页面网址相同的网址时将跳过
+     */
+    nextPageUrl?: (payload: {
+        data: T;
+        /** 当前页面元素 */
+        el: HTMLElement;
+        /** 当前页面网址 */
+        url: string;
+    }) => AsyncOrSync<string | undefined>;
+    /** @inner */
+    _url?: string[];
+    /**
+     * 抓取引擎
+     */
+    engine?: XCrawlerEngine;
+    engineOptions?: XPuppeteerGetPageHtmlOptions;
+    /**
+     * 数据提取
+     */
+    extract?: XCrawlerDataExtract<T>;
+    /**
+     * 下一步
+     */
+    next?: OneOrMore<string | XCrawlerNext<T>>;
+};
+export type XCrawlerConfig<T extends XCrawlerData> = {
+    /**
+     * 入口页链接
+     */
+    entry: string;
+    /**
+     * 使用代理
+     */
+    proxy?: string;
+    /**
+     * 内容类型
+     */
+    contentType?: XCrawlerContentType;
+    /**
+     * 抓取引擎
+     */
+    engine?: XCrawlerEngine;
+    engineOptions?: XPuppeteerGetPageHtmlOptions;
+    /**
+     * 数据格式
+     */
+    schema: (_: typeof v) => VaeObjectSchemaShapeOf<T>;
+    /**
+     * 数据提取
+     */
+    extract?: XCrawlerDataExtract<T>;
+    /**
+     * 下一步
+     */
+    next?: OneOrMore<XCrawlerNext<T>>;
+    /**
+     * 数据监听
+     */
+    listen?: (data: T) => any;
+};
+export declare class XCrawler<T extends XCrawlerData> extends EventBus<{
+    newData: (data: T) => any;
+}> {
+    private config;
+    private schemaMap;
+    private id;
+    constructor(config: XCrawlerConfig<T>);
+    private normalizeUrl;
+    private fetchUrlHeadless;
+    private fetchUrlHeadful;
+    private fetchUrlRequest;
+    private fetchUrl;
+    private fetchUrlAsHTMLElement;
+    private fetchUrlAsJSON;
+    private ocr;
+    private findNextById;
+    private pageUrlCache;
+    next(pageUrl: string, contentType: XCrawlerContentType, engine: XCrawlerEngine, elOrField: any, data: Partial<T>, next: XCrawlerNext<T>, index: number, rootElement: HTMLElement): Promise<void>;
+    start(): Promise<void>;
+}

package/crawler.js

@@ -0,0 +1,332 @@
+import { xr } from "@jayfong/x-request";
+import { parse as parseHTML } from "node-html-parser";
+import {
+  EventBus,
+  castArray,
+  cloneDeepFast,
+  escape,
+  get,
+  isRegExp,
+  isUrl,
+  keysStrict,
+  memoize,
+  omitStrict
+} from "vtils";
+import { v } from "vtils/vae";
+import { XOcr } from "./ocr.js";
+import { XPuppeteer } from "./puppeteer.js";
+class XCrawler extends EventBus {
+  constructor(config) {
+    super();
+    this.config = config;
+    this.schemaMap = config.schema(v);
+    if (config.listen) {
+      this.on("newData", (newData) => {
+        config.listen(newData);
+      });
+    }
+  }
+  schemaMap;
+  id = Date.now().toString(36);
+  normalizeUrl(pageUrl, url) {
+    url = url.trim();
+    return !url ? "" : isUrl(url) ? url : new URL(
+      url.startsWith(".") || url.startsWith("/") ? url : `/${url}`,
+      pageUrl
+    ).toString();
+  }
+  async fetchUrlHeadless(pageUrl, url, options) {
+    const text = await XPuppeteer.getPageHtml(this.normalizeUrl(pageUrl, url), {
+      headful: false,
+      proxy: this.config.proxy,
+      blockAssets: true,
+      instanceId: this.id,
+      ...options
+    });
+    return text;
+  }
+  async fetchUrlHeadful(pageUrl, url, options) {
+    const text = await XPuppeteer.getPageHtml(this.normalizeUrl(pageUrl, url), {
+      headful: true,
+      proxy: this.config.proxy,
+      blockAssets: true,
+      instanceId: this.id,
+      ...options
+    });
+    return text;
+  }
+  async fetchUrlRequest(pageUrl, url) {
+    const absoluteUrl = this.normalizeUrl(pageUrl, url);
+    const { data: text } = await xr.getText({
+      url: absoluteUrl,
+      autoReferer: true,
+      proxyUrl: this.config.proxy
+    });
+    return text;
+  }
+  async fetchUrl(pageUrl, url, engine, options) {
+    const text = engine === "headless" ? await this.fetchUrlHeadless(pageUrl, url, options) : engine === "headful" ? await this.fetchUrlHeadful(pageUrl, url, options) : await this.fetchUrlRequest(pageUrl, url);
+    return text;
+  }
+  async fetchUrlAsHTMLElement(pageUrl, url, engine, options) {
+    const text = await this.fetchUrl(pageUrl, url, engine, options);
+    return parseHTML(text);
+  }
+  async fetchUrlAsJSON(pageUrl, url, engine) {
+    const text = await this.fetchUrl(pageUrl, url, engine);
+    return JSON.parse(text);
+  }
+  async ocr(pageUrl, el) {
+    const imgDocs = el.querySelectorAll("img");
+    for (const imgDoc of imgDocs) {
+      const imgUrl = imgDoc.getAttribute("src");
+      if (imgUrl) {
+        const ocrText = await XOcr.recognize(
+          this.normalizeUrl(pageUrl, imgUrl)
+        );
+        imgDoc.replaceWith(`<div>${escape(ocrText)} </div>`);
+      }
+    }
+  }
+  findNextById = memoize((id) => {
+    const findNextById = (nexts) => {
+      for (const next of castArray(nexts)) {
+        if (typeof next !== "string") {
+          if (next.id === id) {
+            return next;
+          }
+          if (next.next) {
+            const _next = findNextById(next.next);
+            if (_next) {
+              return _next;
+            }
+          }
+        }
+      }
+    };
+    return this.config.next && findNextById(this.config.next);
+  });
+  pageUrlCache = /* @__PURE__ */ new Set();
+  async next(pageUrl, contentType, engine, elOrField, data, next, index, rootElement) {
+    if (next.filter && !await next.filter(data)) {
+      return;
+    }
+    contentType = next.contentType || contentType;
+    engine = next.engine || engine;
+    if (contentType === "json") {
+      const field = elOrField;
+      const nextFields = next.url ? await Promise.all(
+        castArray(
+          typeof next.url === "function" ? await next.url(data) : data[next.url]
+        ).map((url) => this.fetchUrlAsJSON(pageUrl, url, engine))
+      ) : !next.field || next.field === "." ? [field] : typeof next.field === "string" ? get(field, next.field) : castArray(await next.field(field));
+      for (const nextField of nextFields) {
+        const newData = cloneDeepFast(data);
+        if (next.extract) {
+          const fieldKeys = keysStrict(next.extract);
+          for (const fieldKey of fieldKeys) {
+            const fieldConfig = typeof next.extract[fieldKey] === "string" || typeof next.extract[fieldKey] === "function" || Array.isArray(next.extract[fieldKey]) ? { select: next.extract[fieldKey] } : next.extract[fieldKey];
+            let fieldValue = !fieldConfig.select || fieldConfig.select === "." ? nextField : typeof fieldConfig.select === "string" ? get(nextField, fieldConfig.select) : Array.isArray(fieldConfig.select) ? "TODO" : "TODO";
+            if (fieldValue != null) {
+              fieldValue = fieldConfig.transform ? await fieldConfig.transform(fieldValue) : fieldValue;
+              const fieldSchema = this.schemaMap[fieldKey];
+              fieldValue = fieldSchema.cast(fieldValue);
+              newData[fieldKey] = fieldValue;
+            }
+          }
+        }
+        if (next.next) {
+          for (const nextItem of castArray(next.next)) {
+            await this.next(
+              pageUrl,
+              contentType,
+              engine,
+              nextField,
+              newData,
+              typeof nextItem === "string" ? this.findNextById(nextItem) : nextItem,
+              0,
+              rootElement
+            );
+          }
+        } else {
+          this.emit("newData", newData);
+        }
+      }
+    } else {
+      const el = elOrField;
+      const nextEls = next.url ? await Promise.all(
+        next._url.map(
+          (fullUrl) => this.fetchUrlAsHTMLElement(
+            pageUrl,
+            fullUrl,
+            engine,
+            next.engineOptions
+          )
+        )
+      ) : !next.element || next.element === "." ? [el] : typeof next.element === "string" ? el.querySelectorAll(next.element) : castArray(
+        await next.element(el, {
+          index,
+          root: rootElement
+        })
+      );
+      const nextParamsList = [];
+      let nextElIndex = 0;
+      for (const nextEl of nextEls) {
+        const newData = cloneDeepFast(data);
+        if (next.extract) {
+          const fieldKeys = keysStrict(next.extract);
+          for (const fieldKey of fieldKeys) {
+            const fieldConfig = typeof next.extract[fieldKey] === "string" || typeof next.extract[fieldKey] === "function" || Array.isArray(next.extract[fieldKey]) ? { select: next.extract[fieldKey] } : next.extract[fieldKey];
+            const fieldEl = !fieldConfig.select || fieldConfig.select === "." ? nextEl : typeof fieldConfig.select === "string" ? nextEl.querySelector(fieldConfig.select) : Array.isArray(fieldConfig.select) ? nextEl.querySelectorAll(fieldConfig.select[0]).find((item) => {
+              const filter = fieldConfig.select[1];
+              return typeof filter === "string" ? item.text.includes(filter) : isRegExp(filter) ? filter.test(item.text) : filter(item);
+            }) : await fieldConfig.select(nextEl);
+            if (fieldEl != null) {
+              let fieldText = "";
+              if (typeof fieldEl === "string") {
+                fieldText = fieldEl.trim();
+              } else {
+                if (fieldConfig.ocr) {
+                  await this.ocr(pageUrl, fieldEl);
+                }
+                switch (fieldConfig.extract) {
+                  case "getHTML":
+                    fieldText = fieldEl.outerHTML;
+                    break;
+                  case "getFirstImageUrl":
+                    fieldText = this.normalizeUrl(
+                      pageUrl,
+                      fieldEl.getAttribute("src") || fieldEl.querySelector("img")?.getAttribute("src") || ""
+                    );
+                    break;
+                  case "getFirstLinkUrl":
+                    fieldText = this.normalizeUrl(
+                      pageUrl,
+                      fieldEl.getAttribute("href") || fieldEl.querySelector("a")?.getAttribute("href") || ""
+                    );
+                    break;
+                  default:
+                    fieldText = fieldEl.text.trim();
+                    break;
+                }
+              }
+              let fieldValue = fieldConfig.transform ? await fieldConfig.transform(fieldText) : fieldText;
+              const fieldSchema = this.schemaMap[fieldKey];
+              if (fieldSchema.options.type !== "array") {
+                fieldValue = fieldSchema.cast(fieldValue);
+              } else {
+                fieldValue = fieldSchema.cast([
+                  ...newData[fieldKey] || [],
+                  fieldValue
+                ]);
+              }
+              newData[fieldKey] = fieldValue;
+            }
+          }
+        }
+        if (next.next) {
+          const nexts = await Promise.all(
+            castArray(next.next).map(async (nextItem) => {
+              nextItem = cloneDeepFast(
+                typeof nextItem === "string" ? this.findNextById(nextItem) : nextItem
+              );
+              if (nextItem.url) {
+                nextItem._url = castArray(
+                  typeof nextItem.url === "function" ? await nextItem.url({
+                    data: newData,
+                    el: nextEl
+                  }) : newData[nextItem.url]
+                ).map((url) => this.normalizeUrl(pageUrl, url)).filter((fullUrl) => !this.pageUrlCache.has(fullUrl)).map((fullUrl) => {
+                  this.pageUrlCache.add(fullUrl);
+                  return fullUrl;
+                });
+              }
+              return nextItem;
+            })
+          );
+          let i = 0;
+          for (const nextItem of nexts) {
+            nextParamsList.push([
+              nextItem.url && nextItem._url[i] || pageUrl,
+              contentType,
+              engine,
+              nextEl,
+              newData,
+              nextItem,
+              nextElIndex,
+              rootElement
+            ]);
+            i++;
+          }
+        } else {
+          this.emit("newData", newData);
+        }
+        nextElIndex++;
+      }
+      for (const nextParams of nextParamsList) {
+        await this.next(...nextParams);
+        if (next.nextPageUrl) {
+          const nextPageUrl = await next.nextPageUrl({
+            data: nextParams[4],
+            el: nextParams[3],
+            url: nextParams[0]
+          });
+          if (nextPageUrl) {
+            const nextPageAbsoluteUrl = this.normalizeUrl(
+              nextParams[0],
+              nextPageUrl
+            );
+            if (nextPageAbsoluteUrl !== nextParams[0]) {
+              const nextPageElement = await this.fetchUrlAsHTMLElement(
+                nextPageAbsoluteUrl,
+                nextPageAbsoluteUrl,
+                engine,
+                this.config.engineOptions
+              );
+              await this.next(
+                nextPageAbsoluteUrl,
+                contentType,
+                engine,
+                nextPageElement,
+                nextParams[4],
+                omitStrict(next, ["url"]),
+                index,
+                rootElement
+              );
+            }
+          }
+        }
+      }
+    }
+  }
+  async start() {
+    const initialContentType = this.config.contentType || "html";
+    const initialEngine = this.config.engine || "request";
+    const rootElement = initialContentType === "json" ? await this.fetchUrlAsJSON(
+      this.config.entry,
+      this.config.entry,
+      initialEngine
+    ) : await this.fetchUrlAsHTMLElement(
+      this.config.entry,
+      this.config.entry,
+      initialEngine,
+      this.config.engineOptions
+    );
+    await this.next(
+      this.config.entry,
+      initialContentType,
+      initialEngine,
+      rootElement,
+      {},
+      {
+        extract: this.config.extract,
+        next: this.config.next
+      },
+      0,
+      rootElement
+    );
+  }
+}
+export {
+  XCrawler
+};

package/csv.d.ts

@@ -0,0 +1,14 @@
+export type XCsvParseOptions = {
+    arrayFieldNames?: string[];
+    arrayDelimiter?: string;
+    objectFieldNames?: string[];
+    booleanFieldNames?: string[];
+    dateFieldNames?: string[];
+};
+export type XCsvStringifyOptions = XCsvParseOptions;
+export declare class XCsv {
+    static parse<T>(value: string, options?: XCsvParseOptions): Promise<T[]>;
+    static stringify(value: any[], options?: XCsvStringifyOptions): Promise<string>;
+    static read<T>(file: string, options?: XCsvParseOptions): Promise<T[]>;
+    static write(file: string, value: any, options?: XCsvStringifyOptions): Promise<void>;
+}

package/csv.js

@@ -0,0 +1,67 @@
+import * as csv from "csv";
+import * as fs from "node:fs";
+import { mapValues } from "vtils";
+import { formatDate } from "vtils/date";
+class XCsv {
+  static async parse(value, options = {}) {
+    const res = [];
+    for await (const item of csv.parse(value, {
+      columns: true
+    })) {
+      res.push(
+        options.objectFieldNames?.length || options.booleanFieldNames?.length || options.dateFieldNames?.length || options.arrayFieldNames?.length && options.arrayDelimiter ? mapValues(item, (v, k) => {
+          if (options.arrayFieldNames.includes(k)) {
+            return v.split(options.arrayDelimiter).filter(Boolean);
+          }
+          if (options.objectFieldNames?.includes(k)) {
+            return v ? JSON.parse(v) : {};
+          }
+          if (options.booleanFieldNames?.includes(k)) {
+            return v ? v === 1 : void 0;
+          }
+          if (options.dateFieldNames?.includes(k)) {
+            return v ? new Date(v) : void 0;
+          }
+          return v;
+        }) : item
+      );
+    }
+    return res;
+  }
+  static async stringify(value, options = {}) {
+    let res = "";
+    for await (const line of csv.stringify(
+      options.objectFieldNames?.length || options.booleanFieldNames?.length || options.dateFieldNames?.length || options.arrayFieldNames?.length && options.arrayDelimiter ? value.map(
+        (item) => mapValues(item, (v, k) => {
+          if (options.arrayFieldNames.includes(k)) {
+            return v ? v.join(options.arrayDelimiter) : "";
+          }
+          if (options.objectFieldNames?.includes(k)) {
+            return JSON.stringify(v || {});
+          }
+          if (options.booleanFieldNames?.includes(k)) {
+            return v ? "1" : "0";
+          }
+          if (options.dateFieldNames?.includes(k)) {
+            return v ? formatDate(v, "yyyy-mm-dd hh:ii:ss") : v;
+          }
+          return v;
+        })
+      ) : value,
+      { header: true }
+    )) {
+      res += line;
+    }
+    return res;
+  }
+  static async read(file, options = {}) {
+    const value = await fs.promises.readFile(file, "utf8");
+    return XCsv.parse(value, options);
+  }
+  static async write(file, value, options = {}) {
+    await fs.promises.writeFile(file, await XCsv.stringify(value, options));
+  }
+}
+export {
+  XCsv
+};

package/data_set.d.ts

@@ -0,0 +1,22 @@
+import { OmitStrict } from 'vtils/types';
+import { VaeObjectSchemaShapeOf, v } from 'vtils/vae';
+export type XDataSetOptions<T extends Record<string, any>, K extends keyof T> = {
+    storeFile: string;
+    dataSchema: (_: typeof v) => VaeObjectSchemaShapeOf<T>;
+    dataKey: K;
+    csvArrayDelimiter?: string;
+    autoWriteInterval?: number;
+    noAutoWrite?: boolean;
+};
+export declare class XDataSet<T extends Record<string, any>, K extends keyof T> {
+    private options;
+    private storeType;
+    private dataShapeSchema;
+    private dataSchema;
+    private csvOptions;
+    currentMap: Record<T[K], OmitStrict<T, K>>;
+    get current(): T[];
+    constructor(options: XDataSetOptions<T, K>);
+    read(): Promise<T[]>;
+    write(dataSet?: T[]): Promise<void>;
+}