@cyia/crawl 0.0.12 → 0.0.13

package/define.d.ts

@@ -49,6 +49,9 @@ export declare const ActionDefine: v.SchemaWithFallback<v.UnionSchema<[v.ObjectS
         readonly concurrency: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
     }, undefined>, v.ObjectSchema<{
         readonly mode: v.LiteralSchema<"navigation", undefined>;
+    }, undefined>, v.ObjectSchema<{
+        readonly mode: v.LiteralSchema<"waitBodyElements", undefined>;
+        readonly threshold: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
     }, undefined>], undefined>;
 }, undefined>, v.ObjectSchema<{
     readonly type: v.LiteralSchema<"click", undefined>;
@@ -96,6 +99,12 @@ export declare const ActionDefine: v.SchemaWithFallback<v.UnionSchema<[v.ObjectS
         readonly key: v.StringSchema<undefined>;
         readonly method: v.PicklistSchema<["push", "flat-push", "define", "merge"], undefined>;
     }, undefined>], undefined>, undefined>;
+}, undefined>, v.ObjectSchema<{
+    readonly type: v.LiteralSchema<"rawContent", undefined>;
+    readonly output: v.OptionalSchema<v.UnionSchema<[v.StringSchema<undefined>, v.ObjectSchema<{
+        readonly key: v.StringSchema<undefined>;
+        readonly method: v.PicklistSchema<["push", "flat-push", "define", "merge"], undefined>;
+    }, undefined>], undefined>, undefined>;
 }, undefined>, v.GenericSchema<{
     type: "page";
     input: string;
@@ -121,16 +130,21 @@ export declare const ActionDefine: v.SchemaWithFallback<v.UnionSchema<[v.ObjectS
 }, undefined>, v.ObjectSchema<{
     readonly type: v.LiteralSchema<"read-variable", undefined>;
     readonly input: v.StringSchema<undefined>;
+}, undefined>, v.ObjectSchema<{
+    readonly type: v.LiteralSchema<"evaluate", undefined>;
+    readonly fn: v.CustomSchema<(...args: any[]) => any, undefined>;
+    readonly args: v.OptionalSchema<v.ArraySchema<v.AnySchema, undefined>, undefined>;
+    readonly output: v.OptionalSchema<v.UnionSchema<[v.StringSchema<undefined>, v.ObjectSchema<{
+        readonly key: v.StringSchema<undefined>;
+        readonly method: v.PicklistSchema<["push", "flat-push", "define", "merge"], undefined>;
+    }, undefined>], undefined>, undefined>;
 }, undefined>], undefined>, (item: v.OutputDataset<{
     timeout?: number | undefined;
     waitUntil: "load" | "domcontentloaded" | "networkidle0" | "networkidle2";
-    url: (string | {
-        source: "variable";
-        key: (string | string[] | undefined) & (string | string[]);
-    } | undefined) & (string | {
+    url: string | {
         source: "variable";
-        key: (string | string[] | undefined) & (string | string[]);
-    });
+        key: string | string[];
+    };
     type: "goto";
 } | {
     width: number;
@@ -162,6 +176,9 @@ export declare const ActionDefine: v.SchemaWithFallback<v.UnionSchema<[v.ObjectS
         concurrency?: number | undefined;
     } | {
         mode: "navigation";
+    } | {
+        mode: "waitBodyElements";
+        threshold?: number | undefined;
     };
 } | {
     type: "click";
@@ -175,13 +192,10 @@ export declare const ActionDefine: v.SchemaWithFallback<v.UnionSchema<[v.ObjectS
 } | {
     type: "type";
     selector: string;
-    text: (string | {
-        source: "variable";
-        key: (string | string[] | undefined) & (string | string[]);
-    } | undefined) & (string | {
+    text: string | {
         source: "variable";
-        key: (string | string[] | undefined) & (string | string[]);
-    });
+        key: string | string[];
+    };
     delay?: number | undefined;
 } | {
     type: "keypress";
@@ -212,6 +226,12 @@ export declare const ActionDefine: v.SchemaWithFallback<v.UnionSchema<[v.ObjectS
         key: string;
         method: "push" | "flat-push" | "define" | "merge";
     } | undefined;
+} | {
+    type: "rawContent";
+    output?: string | {
+        key: string;
+        method: "push" | "flat-push" | "define" | "merge";
+    } | undefined;
 } | {
     type: "page";
     input: string;
@@ -232,6 +252,14 @@ export declare const ActionDefine: v.SchemaWithFallback<v.UnionSchema<[v.ObjectS
 } | {
     type: "read-variable";
     input: string;
+} | {
+    type: "evaluate";
+    fn: (...args: any[]) => any;
+    args?: any[] | undefined;
+    output?: string | {
+        key: string;
+        method: "push" | "flat-push" | "define" | "merge";
+    } | undefined;
 }, v.NumberIssue | v.BaseIssue<unknown> | v.BooleanIssue | v.StringIssue | v.TupleIssue | v.ObjectIssue | v.LiteralIssue | v.ArrayIssue | v.UnionIssue<v.StringIssue | v.ArrayIssue> | v.UnionIssue<v.StringIssue | v.ObjectIssue | v.LiteralIssue | v.ArrayIssue | v.UnionIssue<v.StringIssue | v.ArrayIssue>> | v.PicklistIssue | v.UnionIssue<v.StringIssue | v.ObjectIssue | v.PicklistIssue> | v.UnionIssue<v.StringIssue | v.TupleIssue> | v.LooseObjectIssue | v.VariantIssue | v.CustomIssue | v.UnionIssue<v.NumberIssue | v.BaseIssue<unknown> | v.BooleanIssue | v.StringIssue | v.TupleIssue | v.ObjectIssue | v.LiteralIssue | v.ArrayIssue | v.UnionIssue<v.StringIssue | v.ArrayIssue> | v.UnionIssue<v.StringIssue | v.ObjectIssue | v.LiteralIssue | v.ArrayIssue | v.UnionIssue<v.StringIssue | v.ArrayIssue>> | v.PicklistIssue | v.UnionIssue<v.StringIssue | v.ObjectIssue | v.PicklistIssue> | v.UnionIssue<v.StringIssue | v.TupleIssue> | v.LooseObjectIssue | v.VariantIssue | v.CustomIssue>> | undefined) => {
     type: "custom";
     config: any;
@@ -274,6 +302,9 @@ export declare const ActionListDefine: v.ArraySchema<v.SchemaWithFallback<v.Unio
         readonly concurrency: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
     }, undefined>, v.ObjectSchema<{
         readonly mode: v.LiteralSchema<"navigation", undefined>;
+    }, undefined>, v.ObjectSchema<{
+        readonly mode: v.LiteralSchema<"waitBodyElements", undefined>;
+        readonly threshold: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
     }, undefined>], undefined>;
 }, undefined>, v.ObjectSchema<{
     readonly type: v.LiteralSchema<"click", undefined>;
@@ -321,6 +352,12 @@ export declare const ActionListDefine: v.ArraySchema<v.SchemaWithFallback<v.Unio
         readonly key: v.StringSchema<undefined>;
         readonly method: v.PicklistSchema<["push", "flat-push", "define", "merge"], undefined>;
     }, undefined>], undefined>, undefined>;
+}, undefined>, v.ObjectSchema<{
+    readonly type: v.LiteralSchema<"rawContent", undefined>;
+    readonly output: v.OptionalSchema<v.UnionSchema<[v.StringSchema<undefined>, v.ObjectSchema<{
+        readonly key: v.StringSchema<undefined>;
+        readonly method: v.PicklistSchema<["push", "flat-push", "define", "merge"], undefined>;
+    }, undefined>], undefined>, undefined>;
 }, undefined>, v.GenericSchema<{
     type: "page";
     input: string;
@@ -346,16 +383,21 @@ export declare const ActionListDefine: v.ArraySchema<v.SchemaWithFallback<v.Unio
 }, undefined>, v.ObjectSchema<{
     readonly type: v.LiteralSchema<"read-variable", undefined>;
     readonly input: v.StringSchema<undefined>;
+}, undefined>, v.ObjectSchema<{
+    readonly type: v.LiteralSchema<"evaluate", undefined>;
+    readonly fn: v.CustomSchema<(...args: any[]) => any, undefined>;
+    readonly args: v.OptionalSchema<v.ArraySchema<v.AnySchema, undefined>, undefined>;
+    readonly output: v.OptionalSchema<v.UnionSchema<[v.StringSchema<undefined>, v.ObjectSchema<{
+        readonly key: v.StringSchema<undefined>;
+        readonly method: v.PicklistSchema<["push", "flat-push", "define", "merge"], undefined>;
+    }, undefined>], undefined>, undefined>;
 }, undefined>], undefined>, (item: v.OutputDataset<{
     timeout?: number | undefined;
     waitUntil: "load" | "domcontentloaded" | "networkidle0" | "networkidle2";
-    url: (string | {
-        source: "variable";
-        key: (string | string[] | undefined) & (string | string[]);
-    } | undefined) & (string | {
+    url: string | {
         source: "variable";
-        key: (string | string[] | undefined) & (string | string[]);
-    });
+        key: string | string[];
+    };
     type: "goto";
 } | {
     width: number;
@@ -387,6 +429,9 @@ export declare const ActionListDefine: v.ArraySchema<v.SchemaWithFallback<v.Unio
         concurrency?: number | undefined;
     } | {
         mode: "navigation";
+    } | {
+        mode: "waitBodyElements";
+        threshold?: number | undefined;
     };
 } | {
     type: "click";
@@ -400,13 +445,10 @@ export declare const ActionListDefine: v.ArraySchema<v.SchemaWithFallback<v.Unio
 } | {
     type: "type";
     selector: string;
-    text: (string | {
-        source: "variable";
-        key: (string | string[] | undefined) & (string | string[]);
-    } | undefined) & (string | {
+    text: string | {
         source: "variable";
-        key: (string | string[] | undefined) & (string | string[]);
-    });
+        key: string | string[];
+    };
     delay?: number | undefined;
 } | {
     type: "keypress";
@@ -437,6 +479,12 @@ export declare const ActionListDefine: v.ArraySchema<v.SchemaWithFallback<v.Unio
         key: string;
         method: "push" | "flat-push" | "define" | "merge";
     } | undefined;
+} | {
+    type: "rawContent";
+    output?: string | {
+        key: string;
+        method: "push" | "flat-push" | "define" | "merge";
+    } | undefined;
 } | {
     type: "page";
     input: string;
@@ -457,6 +505,14 @@ export declare const ActionListDefine: v.ArraySchema<v.SchemaWithFallback<v.Unio
 } | {
     type: "read-variable";
     input: string;
+} | {
+    type: "evaluate";
+    fn: (...args: any[]) => any;
+    args?: any[] | undefined;
+    output?: string | {
+        key: string;
+        method: "push" | "flat-push" | "define" | "merge";
+    } | undefined;
 }, v.NumberIssue | v.BaseIssue<unknown> | v.BooleanIssue | v.StringIssue | v.TupleIssue | v.ObjectIssue | v.LiteralIssue | v.ArrayIssue | v.UnionIssue<v.StringIssue | v.ArrayIssue> | v.UnionIssue<v.StringIssue | v.ObjectIssue | v.LiteralIssue | v.ArrayIssue | v.UnionIssue<v.StringIssue | v.ArrayIssue>> | v.PicklistIssue | v.UnionIssue<v.StringIssue | v.ObjectIssue | v.PicklistIssue> | v.UnionIssue<v.StringIssue | v.TupleIssue> | v.LooseObjectIssue | v.VariantIssue | v.CustomIssue | v.UnionIssue<v.NumberIssue | v.BaseIssue<unknown> | v.BooleanIssue | v.StringIssue | v.TupleIssue | v.ObjectIssue | v.LiteralIssue | v.ArrayIssue | v.UnionIssue<v.StringIssue | v.ArrayIssue> | v.UnionIssue<v.StringIssue | v.ObjectIssue | v.LiteralIssue | v.ArrayIssue | v.UnionIssue<v.StringIssue | v.ArrayIssue>> | v.PicklistIssue | v.UnionIssue<v.StringIssue | v.ObjectIssue | v.PicklistIssue> | v.UnionIssue<v.StringIssue | v.TupleIssue> | v.LooseObjectIssue | v.VariantIssue | v.CustomIssue>> | undefined) => {
     type: "custom";
     config: any;

package/format.d.ts

@@ -2,3 +2,15 @@ export declare function format(rawHtml: string, options: {
     cleanContent?: boolean;
     format: 'html' | 'text' | 'markdown';
 }): string | null | undefined;
+export declare function formatDoc(rawHtml: string): {
+    title: string | null | undefined;
+    content: string | null | undefined;
+    textContent: string | null | undefined;
+    length: number | null | undefined;
+    excerpt: string | null | undefined;
+    byline: string | null | undefined;
+    dir: string | null | undefined;
+    siteName: string | null | undefined;
+    lang: string | null | undefined;
+    publishedTime: string | null | undefined;
+} | null;

package/full-web-request.d.ts

@@ -0,0 +1,18 @@
+import { WebBrowser } from './init';
+export declare class FullWebRequest {
+    #private;
+    config: {
+        url: string;
+        filterLink: (url: string) => boolean;
+    };
+    browser: WebBrowser;
+    dataMap: Map<string, any>;
+    constructor(config: {
+        url: string;
+        filterLink: (url: string) => boolean;
+    });
+    start(): Promise<Map<string, any>>;
+    searchWebOne(url: string, context?: {
+        from: string;
+    }): Promise<void>;
+}

package/index.d.ts

@@ -3,3 +3,4 @@ export * from './define';
 export * from './page';
 export * from './download';
 export * from './format';
+export * from './full-web-request';

package/index.mjs

@@ -1,925 +1,3 @@
-// src/init.ts
-import puppeteer from "puppeteer-core";
-
-// src/page.ts
-import { ElementHandle } from "puppeteer-core";
-
-// src/format.ts
-import { load } from "cheerio";
-import TurndownService from "turndown";
-import { Readability } from "@mozilla/readability";
-import { JSDOM } from "jsdom";
-function format(rawHtml, options) {
-  if (options.cleanContent) {
-    let dom = new JSDOM(rawHtml);
-    let result = new Readability(dom.window.document).parse();
-    if (result) {
-      switch (options.format) {
-        case "html":
-          return result.content;
-        case "text":
-          return result.textContent;
-        case "markdown": {
-          var turndownService = new TurndownService();
-          return turndownService.turndown(result.content);
-        }
-        default:
-          throw "";
-      }
-    }
-    let $ = load(rawHtml, void 0, true);
-    let $body = $("body");
-    $body.find("script,style,iframe,footer,br,hr,svg,header,img").remove();
-    $body.find("*").removeAttr("class");
-    $body.find("*").removeAttr("style");
-    $("*").contents().filter(function() {
-      return this.type === "comment" || this.type === "text" && !this.data.trim();
-    }).remove();
-    $("*").contents().filter(function() {
-      return this.type === "text" && !!this.data.trim();
-    }).text((i, text) => {
-      return text.trim();
-    });
-    if (options.format === "html") {
-      return $body.html();
-    } else if (options.format === "text") {
-      return $body.text();
-    } else if (options.format === "markdown") {
-      var turndownService = new TurndownService();
-      return turndownService.turndown($body.html());
-    }
-  } else {
-    if (options.format === "html") {
-      return rawHtml;
-    } else if (options.format === "text") {
-      let $ = load(rawHtml, void 0, true);
-      let $body = $("body");
-      return $body.text();
-    } else if (options.format === "markdown") {
-      var turndownService = new TurndownService();
-      return turndownService.turndown(rawHtml);
-    }
-  }
-}
-
-// src/page.ts
-import { promise as fastq } from "fastq";
-var WebPage = class {
-  page;
-  browser;
-  parent;
-  #obj = {};
-  constructor(page, browser, parent) {
-    this.page = page;
-    this.browser = browser;
-    this.parent = parent;
-  }
-  ab;
-  timeoutId;
-  setMaxTimeout(timeout) {
-    this.ab = new AbortController();
-    this.timeoutId = setTimeout(() => {
-      this.ab.abort("timeout");
-    }, timeout);
-  }
-  clearTimeout() {
-    clearTimeout(this.timeoutId);
-  }
-  setVariable(key, value) {
-    this.#obj[key] = value;
-  }
-  getVariable(key) {
-    return this.#obj[key];
-  }
-  #navigatePath(page, paths) {
-    let value = void 0;
-    let findValue = false;
-    for (let i = 0; i < paths.length; i++) {
-      const item = paths[i];
-      if (item === "..") {
-        if (!page.parent) {
-          throw new Error("未找到父级");
-        }
-        page = page.parent;
-      } else if (findValue) {
-        if (!value || typeof value !== "object") {
-          throw new Error(`${paths}路径下未找到值`);
-        }
-        value = value[item];
-      } else {
-        value = page.#obj[item];
-        findValue = true;
-      }
-    }
-    return value;
-  }
-  #getValue(value) {
-    if (typeof value === "string") {
-      return value;
-    } else if (value.source === "variable") {
-      if (typeof value.key === "string") {
-        return this.#obj[value.key];
-      } else {
-        return this.#navigatePath(this, value.key);
-      }
-    }
-  }
-  #setOutput(value, output) {
-    if (!output) {
-      return;
-    }
-    if (typeof output === "string") {
-      this.#obj[output] = value;
-    } else {
-      switch (output.method) {
-        case "push": {
-          if (!Array.isArray(this.#obj[output.key])) {
-            throw new Error(`${output.key}不是数组类型`);
-          }
-          this.#obj[output.key] ||= [];
-          this.#obj[output.key].push(value);
-          break;
-        }
-        case "flat-push": {
-          if (!Array.isArray(this.#obj[output.key])) {
-            throw new Error(`${output.key}不是数组类型`);
-          }
-          if (!Array.isArray(value)) {
-            throw new Error(`${JSON.stringify(value)}不是数组类型`);
-          }
-          this.#obj[output.key] ||= [];
-          this.#obj[output.key].push(...value);
-          break;
-        }
-        case "define": {
-          this.#obj[output.key] = value;
-          break;
-        }
-        case "merge": {
-          if (typeof this.#obj[output.key] !== "object") {
-            throw new Error(`${output.key}不是对象类型`);
-          }
-          this.#obj[output.key] ||= {};
-          this.#obj[output.key] = { ...this.#obj[output.key], ...value };
-          break;
-        }
-        default:
-          break;
-      }
-    }
-  }
-  async exeQueue(list) {
-    let value;
-    for (const item of list) {
-      console.log("准备执行", item);
-      switch (item.type) {
-        case "click":
-          await this.page.click(item.selector, {
-            offset: item.offset,
-            delay: item.delay,
-            count: item.count
-          });
-          break;
-        case "type": {
-          await this.page.type(item.selector, this.#getValue(item.text), {
-            delay: item.delay
-          });
-          break;
-        }
-        case "goto": {
-          value = await this.page.goto(this.#getValue(item.url), {
-            waitUntil: item.waitUntil,
-            signal: this.ab?.signal,
-            timeout: this.browser.getConfig()?.actionTimeout
-          });
-          break;
-        }
-        case "setViewport": {
-          value = await this.page.setViewport({
-            width: item.width,
-            height: item.height,
-            isMobile: item.isMobile,
-            isLandscape: item.isLandscape
-          });
-          break;
-        }
-        case "wait": {
-          switch (item.config.mode) {
-            case "selector": {
-              value = await this.page.waitForSelector(item.config.selector, {
-                visible: item.config.visible,
-                hidden: item.config.hidden,
-                signal: this.ab?.signal,
-                timeout: this.browser.getConfig()?.actionTimeout
-              });
-              break;
-            }
-            case "request": {
-              const config = item.config;
-              value = await this.page.waitForRequest(
-                async (req) => {
-                  if (config.urlRegexp) {
-                    let result = config.urlRegexp.test(req.url());
-                    if (!result) {
-                      return false;
-                    }
-                  }
-                  if (config.method && config.method !== req.method()) {
-                    return false;
-                  }
-                  return true;
-                },
-                {
-                  signal: this.ab?.signal,
-                  timeout: this.browser.getConfig()?.actionTimeout
-                }
-              );
-              break;
-            }
-            case "response": {
-              const config = item.config;
-              value = await this.page.waitForResponse(
-                async (res) => {
-                  if (config.urlRegexp) {
-                    let result = config.urlRegexp.test(res.url());
-                    if (!result) {
-                      return false;
-                    }
-                  }
-                  if (config.status && config.status !== res.status()) {
-                    return false;
-                  }
-                  return true;
-                },
-                {
-                  signal: this.ab?.signal,
-                  timeout: this.browser.getConfig()?.actionTimeout
-                }
-              );
-              break;
-            }
-            case "networkIdle": {
-              value = await this.page.waitForNetworkIdle({
-                idleTime: item.config.idleTime,
-                concurrency: item.config.concurrency,
-                signal: this.ab?.signal,
-                timeout: this.browser.getConfig()?.actionTimeout
-              });
-              break;
-            }
-            case "navigation": {
-              value = await this.page.waitForNavigation({
-                signal: this.ab?.signal,
-                timeout: this.browser.getConfig()?.actionTimeout
-              });
-              break;
-            }
-          }
-          break;
-        }
-        case "selector": {
-          if (item.multi) {
-            this.#setOutput(value = await this.page.$$(item.selector), item.output);
-          } else {
-            this.#setOutput(value = await this.page.$(item.selector), item.output);
-          }
-          break;
-        }
-        case "keypress": {
-          await this.page.keyboard.press(item.key, { delay: item.delay });
-          break;
-        }
-        case "findData": {
-          let data = this.#obj[item.input];
-          if (Array.isArray(data)) {
-            if (item.kind === "property") {
-              this.#setOutput(
-                value = await Promise.all(
-                  data.map((el) => {
-                    return el.getProperty(item.key).then((a) => {
-                      return a.jsonValue();
-                    });
-                  })
-                ),
-                item.output
-              );
-            }
-          } else {
-            if (data instanceof ElementHandle) {
-              this.#setOutput(
-                value = data.getProperty(item.key).then((a) => {
-                  return a.jsonValue();
-                }),
-                item.output
-              );
-            }
-          }
-          break;
-        }
-        case "getContent": {
-          let content = await this.page.content();
-          this.#setOutput(value = format(content, { cleanContent: item.cleanContent, format: item.format }), item.output);
-          break;
-        }
-        case "page": {
-          let inputValue = this.#obj[item.input];
-          let list2 = Array.isArray(inputValue) ? inputValue : [inputValue];
-          let queue = fastq(async (input) => {
-            console.log("准备执行", input);
-            try {
-              let result = await this.browser.openPage(async (page) => {
-                page.setVariable("$item", list2[input.index]);
-                page.setVariable("$index", input.index);
-                page.setVariable("$first", input.index === 0);
-                page.setVariable("$last", input.index === list2.length - 1);
-                return await page.exeQueue(item.actions);
-              }, this);
-              resultList.push(result);
-            } catch (error) {
-              if (item.throwError) {
-                throw error;
-              } else {
-                resultList.push(void 0);
-              }
-            }
-          }, item.concurrency);
-          let queueError;
-          queue.error((error) => {
-            if (error) {
-              queueError = error;
-            }
-          });
-          let resultList = [];
-          for (let index = 0; index < list2.length; index++) {
-            queue.push({ index });
-          }
-          await queue.drained();
-          if (item.throwError && queueError) {
-            throw queueError;
-          }
-          value = resultList;
-          this.#setOutput(value, item.output);
-          break;
-        }
-        case "setUserAgent": {
-          await this.page.setUserAgent(item.userAgent);
-          break;
-        }
-        case "close": {
-          await this.page.close({ runBeforeUnload: false });
-          this.clearTimeout();
-          break;
-        }
-        case "custom": {
-          if (typeof item.fn === "function") {
-            value = await item.fn(this);
-          } else {
-            let plugin = this.browser.getCustom(item.config.type);
-            if (!plugin) {
-              throw new Error(`自定义[${item.config.type}]未实现处理`);
-            }
-            value = await plugin(item.config, this);
-          }
-          break;
-        }
-        case "read-variable": {
-          value = this.#obj[item.input];
-          break;
-        }
-        default:
-          break;
-      }
-    }
-    return value;
-  }
-};
-
-// src/define.ts
-import * as v from "valibot";
-var KEYLIST = v.picklist([
-  "0",
-  "1",
-  "2",
-  "3",
-  "4",
-  "5",
-  "6",
-  "7",
-  "8",
-  "9",
-  "Power",
-  "Eject",
-  "Abort",
-  "Help",
-  "Backspace",
-  "Tab",
-  "Numpad5",
-  "NumpadEnter",
-  "Enter",
-  "\r",
-  "\n",
-  "ShiftLeft",
-  "ShiftRight",
-  "ControlLeft",
-  "ControlRight",
-  "AltLeft",
-  "AltRight",
-  "Pause",
-  "CapsLock",
-  "Escape",
-  "Convert",
-  "NonConvert",
-  "Space",
-  "Numpad9",
-  "PageUp",
-  "Numpad3",
-  "PageDown",
-  "End",
-  "Numpad1",
-  "Home",
-  "Numpad7",
-  "ArrowLeft",
-  "Numpad4",
-  "Numpad8",
-  "ArrowUp",
-  "ArrowRight",
-  "Numpad6",
-  "Numpad2",
-  "ArrowDown",
-  "Select",
-  "Open",
-  "PrintScreen",
-  "Insert",
-  "Numpad0",
-  "Delete",
-  "NumpadDecimal",
-  "Digit0",
-  "Digit1",
-  "Digit2",
-  "Digit3",
-  "Digit4",
-  "Digit5",
-  "Digit6",
-  "Digit7",
-  "Digit8",
-  "Digit9",
-  "KeyA",
-  "KeyB",
-  "KeyC",
-  "KeyD",
-  "KeyE",
-  "KeyF",
-  "KeyG",
-  "KeyH",
-  "KeyI",
-  "KeyJ",
-  "KeyK",
-  "KeyL",
-  "KeyM",
-  "KeyN",
-  "KeyO",
-  "KeyP",
-  "KeyQ",
-  "KeyR",
-  "KeyS",
-  "KeyT",
-  "KeyU",
-  "KeyV",
-  "KeyW",
-  "KeyX",
-  "KeyY",
-  "KeyZ",
-  "MetaLeft",
-  "MetaRight",
-  "ContextMenu",
-  "NumpadMultiply",
-  "NumpadAdd",
-  "NumpadSubtract",
-  "NumpadDivide",
-  "F1",
-  "F2",
-  "F3",
-  "F4",
-  "F5",
-  "F6",
-  "F7",
-  "F8",
-  "F9",
-  "F10",
-  "F11",
-  "F12",
-  "F13",
-  "F14",
-  "F15",
-  "F16",
-  "F17",
-  "F18",
-  "F19",
-  "F20",
-  "F21",
-  "F22",
-  "F23",
-  "F24",
-  "NumLock",
-  "ScrollLock",
-  "AudioVolumeMute",
-  "AudioVolumeDown",
-  "AudioVolumeUp",
-  "MediaTrackNext",
-  "MediaTrackPrevious",
-  "MediaStop",
-  "MediaPlayPause",
-  "Semicolon",
-  "Equal",
-  "NumpadEqual",
-  "Comma",
-  "Minus",
-  "Period",
-  "Slash",
-  "Backquote",
-  "BracketLeft",
-  "Backslash",
-  "BracketRight",
-  "Quote",
-  "AltGraph",
-  "Props",
-  "Cancel",
-  "Clear",
-  "Shift",
-  "Control",
-  "Alt",
-  "Accept",
-  "ModeChange",
-  " ",
-  "Print",
-  "Execute",
-  "\0",
-  "a",
-  "b",
-  "c",
-  "d",
-  "e",
-  "f",
-  "g",
-  "h",
-  "i",
-  "j",
-  "k",
-  "l",
-  "m",
-  "n",
-  "o",
-  "p",
-  "q",
-  "r",
-  "s",
-  "t",
-  "u",
-  "v",
-  "w",
-  "x",
-  "y",
-  "z",
-  "Meta",
-  "*",
-  "+",
-  "-",
-  "/",
-  ";",
-  "=",
-  ",",
-  ".",
-  "`",
-  "[",
-  "\\",
-  "]",
-  "'",
-  "Attn",
-  "CrSel",
-  "ExSel",
-  "EraseEof",
-  "Play",
-  "ZoomOut",
-  ")",
-  "!",
-  "@",
-  "#",
-  "$",
-  "%",
-  "^",
-  "&",
-  "(",
-  "A",
-  "B",
-  "C",
-  "D",
-  "E",
-  "F",
-  "G",
-  "H",
-  "I",
-  "J",
-  "K",
-  "L",
-  "M",
-  "N",
-  "O",
-  "P",
-  "Q",
-  "R",
-  "S",
-  "T",
-  "U",
-  "V",
-  "W",
-  "X",
-  "Y",
-  "Z",
-  ":",
-  "<",
-  "_",
-  ">",
-  "?",
-  "~",
-  "{",
-  ",",
-  "}",
-  '"',
-  "SoftLeft",
-  "SoftRight",
-  "Camera",
-  "Call",
-  "EndCall",
-  "VolumeDown",
-  "VolumeUp"
-]);
-var OptNumber = v.optional(v.number());
-var OptBoolean = v.optional(v.boolean());
-var OptString = v.optional(v.string());
-var TimeoutDefine = OptNumber;
-var RegExpStr = v.pipe(
-  v.string(),
-  v.transform((input) => {
-    return new RegExp(input);
-  })
-);
-var RegexpTup = v.pipe(
-  v.tuple([v.string(), v.pipe(v.string())]),
-  v.transform(([input, flag]) => {
-    return new RegExp(input, flag);
-  })
-);
-var RegexpUni = v.union([RegExpStr, RegexpTup]);
-var Selector = v.string();
-var Value = v.union([v.string(), v.object({ source: v.literal("variable"), key: v.union([v.string(), v.array(v.string())]) })]);
-var OutputP = v.optional(
-  v.union([v.string(), v.object({ key: v.string(), method: v.picklist(["push", "flat-push", "define", "merge"]) })])
-);
-var GoToA = v.object({
-  timeout: TimeoutDefine,
-  waitUntil: v.optional(v.picklist(["load", "domcontentloaded", "networkidle0", "networkidle2"]), "networkidle2"),
-  url: Value,
-  type: v.literal("goto")
-});
-var SetViewportA = v.object({
-  width: v.optional(v.number(), 1920),
-  height: v.optional(v.number(), 1080),
-  isMobile: v.optional(v.boolean()),
-  isLandscape: v.optional(v.boolean()),
-  type: v.literal("setViewport")
-});
-var SetUserAgentA = v.object({
-  userAgent: v.string(),
-  type: v.literal("setUserAgent")
-});
-var SelectorCommon = v.object({
-  selector: Selector,
-  visible: OptBoolean,
-  hidden: OptBoolean
-});
-var WaitSelector = v.object({
-  mode: v.literal("selector"),
-  ...SelectorCommon.entries
-});
-var WaitRequest = v.object({
-  mode: v.literal("request"),
-  urlRegexp: RegexpUni,
-  method: OptString
-});
-var WaitResponse = v.object({
-  mode: v.literal("response"),
-  urlRegexp: v.optional(RegexpUni),
-  // method: OptString,
-  status: OptNumber
-});
-var WaitNetworkIdle = v.object({
-  mode: v.literal("networkIdle"),
-  idleTime: OptNumber,
-  concurrency: OptNumber
-});
-var WaitNavigation = v.object({
-  mode: v.literal("navigation")
-});
-var WaitA = v.object({
-  type: v.literal("wait"),
-  config: v.variant("mode", [WaitSelector, WaitRequest, WaitResponse, WaitNetworkIdle, WaitNavigation])
-});
-var ClickA = v.object({
-  type: v.literal("click"),
-  selector: Selector,
-  offset: v.optional(v.object({ x: v.number(), y: v.number() })),
-  delay: OptNumber,
-  count: OptNumber
-});
-var TypeA = v.object({
-  type: v.literal("type"),
-  selector: Selector,
-  text: Value,
-  delay: OptNumber
-});
-var KeyPress = v.object({
-  type: v.literal("keypress"),
-  key: KEYLIST,
-  delay: OptNumber
-});
-var SelectEl = v.object({
-  type: v.literal("selector"),
-  // ...SelectorCommon.entries,
-  selector: Selector,
-  output: OutputP,
-  multi: v.optional(v.boolean(), false)
-});
-var FindData = v.object({
-  type: v.literal("findData"),
-  input: v.string(),
-  output: OutputP,
-  kind: v.picklist(["property"]),
-  key: v.optional(v.string())
-  // multi: v.optional(v.boolean(), true),
-});
-var GetContent = v.object({
-  type: v.literal("getContent"),
-  format: v.optional(v.picklist(["html", "text", "markdown"]), "html"),
-  cleanContent: OptBoolean,
-  output: OutputP
-});
-var CloseA = v.object({
-  type: v.literal("close")
-});
-var PluginA = v.object({
-  type: v.literal("custom"),
-  config: v.optional(v.looseObject({ type: v.string() })),
-  fn: v.optional(v.custom(Boolean))
-});
-var ReadVariable = v.object({
-  type: v.literal("read-variable"),
-  input: v.string()
-});
-var PageA = v.object({
-  type: v.literal("page"),
-  input: v.string(),
-  output: OutputP,
-  concurrency: v.optional(v.number(), 2),
-  throwError: v.optional(v.boolean(), false),
-  actions: v.lazy(() => v.array(ActionDefine))
-});
-var TypeList = [
-  ...[
-    GoToA,
-    SetViewportA,
-    SetUserAgentA,
-    WaitA,
-    ClickA,
-    TypeA,
-    KeyPress,
-    SelectEl,
-    FindData,
-    GetContent,
-    // PageA,
-    CloseA,
-    PluginA,
-    ReadVariable
-  ].map((item) => item.entries.type.literal),
-  "page"
-];
-var ActionDefine = v.fallback(
-  v.union([
-    GoToA,
-    SetViewportA,
-    SetUserAgentA,
-    WaitA,
-    ClickA,
-    TypeA,
-    KeyPress,
-    SelectEl,
-    FindData,
-    GetContent,
-    PageA,
-    CloseA,
-    PluginA,
-    ReadVariable
-  ]),
-  (item) => {
-    if (TypeList.includes((item?.value).type)) {
-      throw new Error(JSON.stringify(item?.issues));
-    }
-    return { type: "custom", config: item?.value };
-  }
-);
-var ActionListDefine = v.array(ActionDefine);
-var GlobalConfig = v.object({
-  maxTimeout: OptNumber,
-  actionTimeout: OptNumber
-});
-
-// src/init.ts
-import * as v2 from "valibot";
-import { Browser as BV, computeExecutablePath as computeExecutablePath2 } from "@puppeteer/browsers";
-import * as fs from "fs";
-
-// src/download.ts
-import { Browser, computeExecutablePath, install } from "@puppeteer/browsers";
-async function download(options) {
-  let result = await install({
-    browser: Browser.CHROME,
-    baseUrl: "https://cdn.npmmirror.com/binaries/chrome-for-testing",
-    ...options,
-    unpack: true
-  });
-}
-function getExecutablePath(dir, buildId) {
-  return computeExecutablePath({ cacheDir: dir, browser: Browser.CHROME, buildId });
-}
-
-// src/init.ts
-import { PUPPETEER_REVISIONS } from "puppeteer-core/internal/revisions.js";
-async function init(options) {
-  return WebBrowser.init(options);
-}
-var CHROME_VERSION = PUPPETEER_REVISIONS.chrome;
-var WebBrowser = class _WebBrowser {
-  browser;
-  static async init(options) {
-    let bvType = BV.CHROME;
-    let executablePath = computeExecutablePath2({ cacheDir: options.cacheDir, browser: bvType, buildId: CHROME_VERSION });
-    if (!fs.existsSync(executablePath)) {
-      console.log("准备下载");
-      await download({ cacheDir: options.cacheDir, buildId: CHROME_VERSION, browser: bvType });
-    }
-    const browser = await puppeteer.launch({ ...options, executablePath });
-    return new _WebBrowser(browser);
-  }
-  constructor(browser) {
-    this.browser = browser;
-  }
-  #config;
-  #pluginMap = /* @__PURE__ */ new Map();
-  setConfig(config) {
-    this.#config = config;
-  }
-  getConfig() {
-    return this.#config;
-  }
-  registerCustom(type, fn) {
-    this.#pluginMap.set(type, fn);
-  }
-  clearCustom() {
-    this.#pluginMap.clear();
-  }
-  getCustom(key) {
-    return this.#pluginMap.get(key);
-  }
-  async openPage(fn, parent) {
-    let page = new WebPage(await this.browser.newPage(), this, parent);
-    if (this.#config?.maxTimeout) {
-      page.setMaxTimeout(this.#config.maxTimeout);
-    }
-    return fn(page);
-  }
-  runQueue(list, input) {
-    let result = v2.safeParse(ActionListDefine, list);
-    if (!result.success) {
-      throw new Error(`解析配置错误
-${JSON.stringify(result.issues)}`);
-    }
-    return this.openPage(async (page) => {
-      if (input) {
-        for (const key in input) {
-          page.setVariable(key, input[key]);
-        }
-      }
-      return page.exeQueue(result.output);
-    });
-  }
-};
-export {
-  ActionDefine,
-  ActionListDefine,
-  GlobalConfig,
-  WebBrowser,
-  WebPage,
-  download,
-  format,
-  getExecutablePath,
-  init
-};
+import ye from"puppeteer-core";import{ElementHandle as z}from"puppeteer-core";import{load as v}from"cheerio";import d from"turndown";import{Readability as k}from"@mozilla/readability";import{JSDOM as x}from"jsdom";function A(n,r){if(r.cleanContent){let t=new x(n),i=new k(t.window.document).parse();if(i)switch(r.format){case"html":return i.content;case"text":return i.textContent;case"markdown":{var o=new d;return o.turndown(i.content)}default:throw""}let a=v(n,void 0,!0),s=a("body");if(s.find("script,style,iframe,footer,br,hr,svg,header,img").remove(),s.find("*").removeAttr("class"),s.find("*").removeAttr("style"),a("*").contents().filter(function(){return this.type==="comment"||this.type==="text"&&!this.data.trim()}).remove(),a("*").contents().filter(function(){return this.type==="text"&&!!this.data.trim()}).text((p,u)=>u.trim()),r.format==="html")return s.html();if(r.format==="text")return s.text();if(r.format==="markdown"){var o=new d;return o.turndown(s.html())}}else{if(r.format==="html")return n;if(r.format==="text")return v(n,void 0,!0)("body").text();if(r.format==="markdown"){var o=new d;return o.turndown(n)}}}function C(n){let r=new x(n);return new k(r.window.document,{charThreshold:100}).parse()}import{promise as X}from"fastq";var g=class{page;browser;parent;#e={};constructor(r,o,t){this.page=r,this.browser=o,this.parent=t}ab;timeoutId;setMaxTimeout(r){this.ab=new AbortController,this.timeoutId=setTimeout(()=>{this.ab.abort("timeout")},r)}clearTimeout(){clearTimeout(this.timeoutId)}setVariable(r,o){this.#e[r]=o}getVariable(r){return this.#e[r]}#r(r,o){let t,i=!1;for(let a=0;a<o.length;a++){let s=o[a];if(s===".."){if(!r.parent)throw new Error("未找到父级");r=r.parent}else if(i){if(!t||typeof t!="object")throw new Error(`${o}路径下未找到值`);t=t[s]}else t=r.#e[s],i=!0}return t}#o(r){if(typeof r=="string")return r;if(r.source==="variable")return typeof r.key=="string"?this.#e[r.key]:this.#r(this,r.key)}#t(r,o){if(o)if(typeof o=="string")this.#e[o]=r;else switch(o.method){case"push":{if(!Array.isArray(this.#e[o.key]))throw new Error(`${o.key}不是数组类型`);this.#e[o.key]||=[],this.#e[o.key].push(r);break}case"flat-push":{if(!Array.isArray(this.#e[o.key]))throw new Error(`${o.key}不是数组类型`);if(!Array.isArray(r))throw new Error(`${JSON.stringify(r)}不是数组类型`);this.#e[o.key]||=[],this.#e[o.key].push(...r);break}case"define":{this.#e[o.key]=r;break}case"merge":{if(typeof this.#e[o.key]!="object")throw new Error(`${o.key}不是对象类型`);this.#e[o.key]||={},this.#e[o.key]={...this.#e[o.key],...r};break}default:break}}async exeQueue(r){let o;for(let t of r)switch(console.log("准备执行",t),t.type){case"click":await this.page.click(t.selector,{offset:t.offset,delay:t.delay,count:t.count});break;case"type":{await this.page.type(t.selector,this.#o(t.text),{delay:t.delay});break}case"goto":{o=await this.page.goto(this.#o(t.url),{waitUntil:t.waitUntil,signal:this.ab?.signal,timeout:this.browser.getConfig()?.actionTimeout});break}case"setViewport":{o=await this.page.setViewport({width:t.width,height:t.height,isMobile:t.isMobile,isLandscape:t.isLandscape});break}case"wait":{switch(t.config.mode){case"selector":{o=await this.page.waitForSelector(t.config.selector,{visible:t.config.visible,hidden:t.config.hidden,signal:this.ab?.signal,timeout:this.browser.getConfig()?.actionTimeout});break}case"request":{let i=t.config;o=await this.page.waitForRequest(async a=>!(i.urlRegexp&&!i.urlRegexp.test(a.url())||i.method&&i.method!==a.method()),{signal:this.ab?.signal,timeout:this.browser.getConfig()?.actionTimeout});break}case"response":{let i=t.config;o=await this.page.waitForResponse(async a=>!(i.urlRegexp&&!i.urlRegexp.test(a.url())||i.status&&i.status!==a.status()),{signal:this.ab?.signal,timeout:this.browser.getConfig()?.actionTimeout});break}case"networkIdle":{o=await this.page.waitForNetworkIdle({idleTime:t.config.idleTime,concurrency:t.config.concurrency,signal:this.ab?.signal,timeout:this.browser.getConfig()?.actionTimeout});break}case"navigation":{o=await this.page.waitForNavigation({signal:this.ab?.signal,timeout:this.browser.getConfig()?.actionTimeout});break}case"waitBodyElements":{let i=t.config.threshold||100;o=await this.page.waitForFunction(a=>document.body.querySelectorAll("*").length>=a,{timeout:this.browser.getConfig()?.actionTimeout,signal:this.ab?.signal},i);break}}break}case"selector":{t.multi?this.#t(o=await this.page.$$(t.selector),t.output):this.#t(o=await this.page.$(t.selector),t.output);break}case"keypress":{await this.page.keyboard.press(t.key,{delay:t.delay});break}case"findData":{let i=this.#e[t.input];Array.isArray(i)?t.kind==="property"&&this.#t(o=await Promise.all(i.map(a=>a.getProperty(t.key).then(s=>s.jsonValue()))),t.output):i instanceof z&&this.#t(o=i.getProperty(t.key).then(a=>a.jsonValue()),t.output);break}case"getContent":{let i=await this.page.content();this.#t(o=A(i,{cleanContent:t.cleanContent,format:t.format}),t.output);break}case"rawContent":{let i=await this.page.content();this.#t(o=i,t.output);break}case"page":{let i=this.#e[t.input],a=Array.isArray(i)?i:[i],s=X(async l=>{console.log("准备执行",l);try{let y=await this.browser.openPage(async f=>(f.setVariable("$item",a[l.index]),f.setVariable("$index",l.index),f.setVariable("$first",l.index===0),f.setVariable("$last",l.index===a.length-1),{result:await f.exeQueue(t.actions),page:f}),this);u.push(y)}catch(y){if(t.throwError)throw y;u.push(void 0)}},t.concurrency),p;s.error(l=>{l&&(p=l)});let u=[];for(let l=0;l<a.length;l++)s.push({index:l});if(await s.drained(),t.throwError&&p)throw p;o=u,this.#t(o,t.output);break}case"setUserAgent":{await this.page.setUserAgent({userAgent:t.userAgent});break}case"close":{await this.page.close({runBeforeUnload:!1}),this.clearTimeout();break}case"custom":{if(typeof t.fn=="function")o=await t.fn(this);else{let i=this.browser.getCustom(t.config.type);if(!i)throw new Error(`自定义[${t.config.type}]未实现处理`);o=await i(t.config,this)}break}case"evaluate":{this.#t(o=await this.page.evaluate(t.fn,...t.args??[]),t.output);break}case"read-variable":{o=this.#e[t.input];break}default:break}return o}dispose(){return this.page.close()}};import*as e from"valibot";var ee=e.picklist(["0","1","2","3","4","5","6","7","8","9","Power","Eject","Abort","Help","Backspace","Tab","Numpad5","NumpadEnter","Enter","\r",`
+`,"ShiftLeft","ShiftRight","ControlLeft","ControlRight","AltLeft","AltRight","Pause","CapsLock","Escape","Convert","NonConvert","Space","Numpad9","PageUp","Numpad3","PageDown","End","Numpad1","Home","Numpad7","ArrowLeft","Numpad4","Numpad8","ArrowUp","ArrowRight","Numpad6","Numpad2","ArrowDown","Select","Open","PrintScreen","Insert","Numpad0","Delete","NumpadDecimal","Digit0","Digit1","Digit2","Digit3","Digit4","Digit5","Digit6","Digit7","Digit8","Digit9","KeyA","KeyB","KeyC","KeyD","KeyE","KeyF","KeyG","KeyH","KeyI","KeyJ","KeyK","KeyL","KeyM","KeyN","KeyO","KeyP","KeyQ","KeyR","KeyS","KeyT","KeyU","KeyV","KeyW","KeyX","KeyY","KeyZ","MetaLeft","MetaRight","ContextMenu","NumpadMultiply","NumpadAdd","NumpadSubtract","NumpadDivide","F1","F2","F3","F4","F5","F6","F7","F8","F9","F10","F11","F12","F13","F14","F15","F16","F17","F18","F19","F20","F21","F22","F23","F24","NumLock","ScrollLock","AudioVolumeMute","AudioVolumeDown","AudioVolumeUp","MediaTrackNext","MediaTrackPrevious","MediaStop","MediaPlayPause","Semicolon","Equal","NumpadEqual","Comma","Minus","Period","Slash","Backquote","BracketLeft","Backslash","BracketRight","Quote","AltGraph","Props","Cancel","Clear","Shift","Control","Alt","Accept","ModeChange"," ","Print","Execute","\0","a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z","Meta","*","+","-","/",";","=",",",".","`","[","\\","]","'","Attn","CrSel","ExSel","EraseEof","Play","ZoomOut",")","!","@","#","$","%","^","&","(","A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z",":","<","_",">","?","~","{",",","}",'"',"SoftLeft","SoftRight","Camera","Call","EndCall","VolumeDown","VolumeUp"]),c=e.optional(e.number()),b=e.optional(e.boolean()),te=e.optional(e.string());var re=c,oe=e.pipe(e.string(),e.transform(n=>new RegExp(n))),ie=e.pipe(e.tuple([e.string(),e.pipe(e.string())]),e.transform(([n,r])=>new RegExp(n,r))),E=e.union([oe,ie]),h=e.string(),P=e.union([e.string(),e.object({source:e.literal("variable"),key:e.union([e.string(),e.array(e.string())])})]),m=e.optional(e.union([e.string(),e.object({key:e.string(),method:e.picklist(["push","flat-push","define","merge"])})])),T=e.object({timeout:re,waitUntil:e.optional(e.picklist(["load","domcontentloaded","networkidle0","networkidle2"]),"networkidle2"),url:P,type:e.literal("goto")}),I=e.object({width:e.optional(e.number(),1920),height:e.optional(e.number(),1080),isMobile:e.optional(e.boolean()),isLandscape:e.optional(e.boolean()),type:e.literal("setViewport")}),O=e.object({userAgent:e.string(),type:e.literal("setUserAgent")}),ne=e.object({selector:h,visible:b,hidden:b}),ae=e.object({mode:e.literal("selector"),...ne.entries}),se=e.object({mode:e.literal("request"),urlRegexp:E,method:te}),le=e.object({mode:e.literal("response"),urlRegexp:e.optional(E),status:c}),ce=e.object({mode:e.literal("networkIdle"),idleTime:c,concurrency:c}),ue=e.object({mode:e.literal("waitBodyElements"),threshold:c}),pe=e.object({mode:e.literal("navigation")}),D=e.object({type:e.literal("wait"),config:e.variant("mode",[ae,se,le,ce,pe,ue])}),S=e.object({type:e.literal("click"),selector:h,offset:e.optional(e.object({x:e.number(),y:e.number()})),delay:c,count:c}),R=e.object({type:e.literal("type"),selector:h,text:P,delay:c}),F=e.object({type:e.literal("keypress"),key:ee,delay:c}),V=e.object({type:e.literal("selector"),selector:h,output:m,multi:e.optional(e.boolean(),!1)}),j=e.object({type:e.literal("findData"),input:e.string(),output:m,kind:e.picklist(["property"]),key:e.optional(e.string())}),N=e.object({type:e.literal("getContent"),format:e.optional(e.picklist(["html","text","markdown"]),"html"),cleanContent:b,output:m}),L=e.object({type:e.literal("rawContent"),output:m}),M=e.object({type:e.literal("evaluate"),fn:e.custom(n=>typeof n=="function"),args:e.optional(e.array(e.any())),output:m}),K=e.object({type:e.literal("close")}),U=e.object({type:e.literal("custom"),config:e.optional(e.looseObject({type:e.string()})),fn:e.optional(e.custom(Boolean))}),B=e.object({type:e.literal("read-variable"),input:e.string()}),fe=e.object({type:e.literal("page"),input:e.string(),output:m,concurrency:e.optional(e.number(),2),throwError:e.optional(e.boolean(),!1),actions:e.lazy(()=>e.array(W))}),me=[...[T,I,O,D,S,R,F,V,j,N,L,K,U,B,M].map(n=>n.entries.type.literal),"page"],W=e.fallback(e.union([T,I,O,D,S,R,F,V,j,N,L,fe,K,U,B,M]),n=>{if(me.includes((n?.value).type))throw new Error(JSON.stringify(n?.issues));return{type:"custom",config:n?.value}}),$=e.array(W),Se=e.object({maxTimeout:c,actionTimeout:c});import*as J from"valibot";import{Browser as de,computeExecutablePath as be}from"@puppeteer/browsers";import*as H from"fs";import{Browser as G,computeExecutablePath as ge,install as he}from"@puppeteer/browsers";async function q(n){let r=await he({browser:G.CHROME,baseUrl:"https://cdn.npmmirror.com/binaries/chrome-for-testing",...n,unpack:!0})}function je(n,r){return ge({cacheDir:n,browser:G.CHROME,buildId:r})}import{PUPPETEER_REVISIONS as we}from"puppeteer-core/internal/revisions.js";async function _(n){return w.init(n)}var Q=we.chrome,w=class n{browser;static async init(r){let o=de.CHROME,t=be({cacheDir:r.cacheDir,browser:o,buildId:Q});H.existsSync(t)||(console.log("准备下载"),await q({cacheDir:r.cacheDir,buildId:Q,browser:o}));let i=await ye.launch({...r,executablePath:t});return new n(i)}constructor(r){this.browser=r}#e;#r=new Map;setConfig(r){this.#e=r}getConfig(){return this.#e}registerCustom(r,o){this.#r.set(r,o)}clearCustom(){this.#r.clear()}getCustom(r){return this.#r.get(r)}async openPage(r,o){let t=new g(await this.browser.newPage(),this,o);return this.#e?.maxTimeout&&t.setMaxTimeout(this.#e.maxTimeout),r(t)}runQueue(r,o){let t=J.safeParse($,r);if(!t.success)throw new Error(`解析配置错误
+${JSON.stringify(t.issues)}`);return this.openPage(async i=>{if(o)for(let a in o)i.setVariable(a,o[a]);return{result:await i.exeQueue(t.output),page:i}})}};import{load as ve}from"cheerio";function Y(n,r){let o=ve(r,{baseURI:n});return o("a").map((i,a)=>o(a).attr("href")?o(a).prop("href"):"").get().filter(Boolean)}var Z=class{config;browser;dataMap=new Map;#e=new Set;constructor(r){this.config=r}async start(){return this.browser=await _({cacheDir:process.cwd(),headless:!1}),this.browser.setConfig({maxTimeout:12e4,actionTimeout:12e4}),await this.searchWebOne(this.config.url,void 0),await this.browser.browser.close(),this.dataMap}async searchWebOne(r,o){let t=await this.browser.runQueue([{type:"setViewport",width:1920,height:1080},{type:"goto",url:r,waitUntil:"networkidle0"},{type:"evaluate",output:"baseURI",fn:()=>window.location.origin},{type:"evaluate",output:"href",fn:()=>window.location.href},{type:"rawContent",output:"data"}]);console.log("解析完成",r);let i=t.page.getVariable("href"),a=t.page.getVariable("baseURI"),s=t.page.getVariable("data"),p=C(s);if(this.#e.add(r),this.#e.add(i),p)this.dataMap.set(r,{requestUrl:r,parsedUrl:i,parent:o?.from,metadata:p,raw:s}),await t.page.dispose();else{this.dataMap.set(r,{requestUrl:r,parsedUrl:i,parent:o?.from,metadata:void 0,raw:s}),await t.page.dispose();return}let u=Y(a,s);u=u.filter(this.config.filterLink);for(let l of u){if(this.#e.has(l)){console.log("已索引,跳过",l);continue}await this.searchWebOne(l,{from:r})}}};export{W as ActionDefine,$ as ActionListDefine,Z as FullWebRequest,Se as GlobalConfig,w as WebBrowser,g as WebPage,q as download,A as format,C as formatDoc,je as getExecutablePath,_ as init};

package/init.d.ts

@@ -20,6 +20,9 @@ export declare class WebBrowser {
     clearCustom(): void;
     getCustom(key: string): PluginFn | undefined;
     openPage<T>(fn: (page: WebPage) => Promise<T>, parent?: WebPage): Promise<T>;
-    runQueue(list: QueueList, input?: Record<string, any>): Promise<any>;
+    runQueue(list: QueueList, input?: Record<string, any>): Promise<{
+        result: any;
+        page: WebPage;
+    }>;
 }
 export {};

package/package.json

@@ -1,22 +1,24 @@
 {
   "name": "@cyia/crawl",
-  "version": "0.0.12",
+  "version": "0.0.13",
   "author": "wszgrcy",
   "description": "",
   "dependencies": {
-    "cheerio": "1.0.0",
-    "fastq": "1.19.1",
-    "html-entities": "2.6.0",
-    "puppeteer-core": "24.6.0",
-    "valibot": "1.0.0",
-    "turndown": "^7.2.0",
     "@mozilla/readability": "^0.6.0",
-    "jsdom": "^26.0.0"
+    "cheerio": "^1.2.0",
+    "fastq": "^1.20.1",
+    "html-entities": "^2.6.0",
+    "htmlparser2": "^10.1.0",
+    "jsdom": "^27.4.0",
+    "puppeteer-core": "^24.36.0",
+    "turndown": "^7.2.2",
+    "valibot": "^1.2.0"
   },
   "exports": {
     ".": {
       "types": "./index.d.ts",
-      "default": "./index.mjs"
+      "default": "./index.mjs",
+      "node": "./index.mjs"
     }
   },
   "publishConfig": {

package/page.d.ts

@@ -15,4 +15,5 @@ export declare class WebPage {
     setVariable(key: string, value: any): void;
     getVariable(key: string): any;
     exeQueue(list: v.InferOutput<typeof ActionDefine>[]): Promise<any>;
+    dispose(): Promise<void>;
 }

package/util/get-page-link.d.ts

@@ -0,0 +1 @@
+export declare function getPageLinks(baseURI: string, content: string): string[];