@letsscrapedata/scraper 0.0.74 → 0.0.76

package/dist/index.d.cts

@@ -2,7 +2,412 @@ import { BrowserControllerType, LsdBrowserType, LsdLaunchOptions, LsdConnectOpti
 import { Proxy } from '@letsscrapedata/proxy';
 import { LogFunction } from '@letsscrapedata/utils';
 
+interface XmlParaCfg {
+    paraname: string;
+    name: string;
+    desc: string;
+    uitype: string;
+    defval: string;
+    listid: string;
+    listparas: string;
+    min: string;
+    max: string;
+    pattern: string;
+    alert: string;
+}
+interface XmlActionBreakCfg {
+    type: string;
+    id: string;
+}
+interface XmlActionCaptchaCfg {
+    type: string;
+    try: string;
+    errname: string;
+    id: string;
+}
+interface XmlActionClickCfg {
+    type: string;
+    wait: string;
+    gen: boolean;
+    cap: boolean;
+    clicktype: string;
+    try: string;
+    errname: string;
+    datapage: string;
+    popupsubtask: boolean;
+    login: boolean;
+    captcha: boolean;
+    navigate: boolean;
+    eurl: string;
+    eloc: string;
+    pn1: string;
+    pv1: string;
+    pn2: string;
+    pv2: string;
+    downloadpath: string;
+    filename: string;
+    pathvarname: string;
+    id: string;
+}
+interface XmlActionContinueCfg {
+    type: string;
+    id: string;
+}
+interface XmlActionMiscCfg {
+    type: string;
+    id: string;
+}
+interface XmlActionExitCfg {
+    type: string;
+    errname: string;
+    id: string;
+}
+interface XmlActionExtractCfg {
+    type: string;
+    tabname: string;
+    id: string;
+}
+interface XmlActionGotoCfg {
+    type: string;
+    url: string;
+    reuse: boolean;
+    wait: string;
+    encodeuri: boolean;
+    gen: boolean;
+    cap: boolean;
+    datapage: string;
+    popupsubtask: boolean;
+    login: boolean;
+    captcha: boolean;
+    eurl: string;
+    eloc: string;
+    pn1: string;
+    pv1: string;
+    pn2: string;
+    pv2: string;
+    headers: string;
+    referer: string;
+    id: string;
+}
+interface XmlActionHoverCfg {
+    type: string;
+    try: string;
+    gen: boolean;
+    cap: boolean;
+    wait: string;
+    errname: string;
+    id: string;
+}
+interface XmlActionIfelseCfg {
+    type: string;
+    id: string;
+}
+interface XmlActionInputCfg {
+    type: string;
+    content: string;
+    enter: boolean;
+    replace: boolean;
+    gen: boolean;
+    cap: boolean;
+    try: string;
+    wait: string;
+    errname: string;
+    datapage: string;
+    popupsubtask: boolean;
+    login: boolean;
+    captcha: boolean;
+    eurl: string;
+    eloc: string;
+    pn1: string;
+    pv1: string;
+    pn2: string;
+    pv2: string;
+    id: string;
+}
+interface XmlActionInterceptClearCfg {
+    type: string;
+    subtype: string;
+}
+interface XmlActionInterceptSetCfg {
+    type: string;
+    subtype: string;
+}
+interface XmlActionLoopdowhileElementCfg {
+    type: string;
+    subtype: string;
+    iswhile: boolean;
+    varname: string;
+    maxloops: string;
+    click: boolean;
+    navigate: boolean;
+    gen: boolean;
+    cap: boolean;
+    errname: string;
+    wait: string;
+    id: string;
+}
+interface XmlActionLoopdowhileTemplstrCfg {
+    type: string;
+    subtype: string;
+    iswhile: boolean;
+    varname: string;
+    maxloops: string;
+    id: string;
+}
+interface XmlActionLoopforCfg {
+    type: string;
+    from: string;
+    to: string;
+    step: string;
+    roundtype: string;
+    varname: string;
+    maxloops: string;
+    errname: string;
+    id: string;
+}
+interface XmlActionLoopinelesCfg {
+    type: string;
+    varname: string;
+    maxloops: string;
+    start: string;
+    end: string;
+    step: string;
+    errname: string;
+    id: string;
+}
+interface XmlActionLoopinstrCfg {
+    type: string;
+    list: string;
+    split: string;
+    varname: string;
+    maxloops: string;
+    trim: boolean;
+    errname: string;
+    id: string;
+}
+interface XmlActionScrollByCfg {
+    type: string;
+    subtype: string;
+    height: string;
+    unit: string;
+    maxtimes: string;
+    interval: string;
+    gen: boolean;
+    cap: boolean;
+    datapage: string;
+    popupsubtask: boolean;
+    login: boolean;
+    captcha: boolean;
+    eurl: string;
+    eloc: string;
+    pn1: string;
+    pv1: string;
+    pn2: string;
+    pv2: string;
+    id: string;
+}
+interface XmlActionScrollIntoviewCfg {
+    type: string;
+    subtype: string;
+    gen: boolean;
+    cap: boolean;
+    errname: string;
+    datapage: string;
+    popupsubtask: boolean;
+    login: boolean;
+    captcha: boolean;
+    eurl: string;
+    eloc: string;
+    pn1: string;
+    pv1: string;
+    pn2: string;
+    pv2: string;
+    id: string;
+}
+interface XmlActionScrollToCfg {
+    type: string;
+    subtype: string;
+    height: string;
+    unit: string;
+    gen: boolean;
+    cap: boolean;
+    datapage: string;
+    popupsubtask: boolean;
+    login: boolean;
+    captcha: boolean;
+    eurl: string;
+    eloc: string;
+    pn1: string;
+    pv1: string;
+    pn2: string;
+    pv2: string;
+    id: string;
+}
+interface XmlActionSelectCfg {
+    type: string;
+    selecttype: string;
+    selectval: string;
+    gen: boolean;
+    cap: boolean;
+    try: string;
+    wait: string;
+    errname: string;
+    datapage: string;
+    popupsubtask: boolean;
+    login: boolean;
+    captcha: boolean;
+    eurl: string;
+    eloc: string;
+    pn1: string;
+    pv1: string;
+    pn2: string;
+    pv2: string;
+    id: string;
+}
+interface XmlActionSetvarDbqueryCfg {
+    type: string;
+    subtype: string;
+    varname: string;
+    defaultval: string;
+    valerrname: string;
+    pattern: string;
+    flags: string;
+    path: string;
+    id: string;
+}
+interface XmlActionSetvarElementCfg {
+    type: string;
+    subtype: string;
+    varname: string;
+    defaultval: string;
+    try: string;
+    valerrname: string;
+    pattern: string;
+    flags: string;
+    path: string;
+    id: string;
+}
+interface XmlActionSetvarFileCfg {
+    type: string;
+    subtype: string;
+    varname: string;
+    defaultval: string;
+    proxy: boolean;
+    valerrname: string;
+    pattern: string;
+    flags: string;
+    path: string;
+    id: string;
+}
+interface XmlActionSetvarGetCfg {
+    type: string;
+    subtype: string;
+    varname: string;
+    defaultval: string;
+    valerrname: string;
+    pattern: string;
+    flags: string;
+    path: string;
+    id: string;
+}
+interface XmlActionSetvarOcrCfg {
+    type: string;
+    subtype: string;
+    varname: string;
+    defaultval: string;
+    valerrname: string;
+    pattern: string;
+    flags: string;
+    path: string;
+    id: string;
+}
+interface XmlActionSetvarSubtaskCfg {
+    type: string;
+    subtype: string;
+    varname: string;
+    defaultval: string;
+    valerrname: string;
+    pattern: string;
+    flags: string;
+    path: string;
+    id: string;
+}
+interface XmlActionSetvarTemplstrCfg {
+    type: string;
+    subtype: string;
+    varname: string;
+    defaultval: string;
+    valerrname: string;
+    pattern: string;
+    flags: string;
+    path: string;
+    id: string;
+}
+interface XmlActionSubtaskCfg {
+    type: string;
+    subtasks: string;
+    sameasparent: boolean;
+    id: string;
+}
+interface XmlActionWaitElementCfg {
+    type: string;
+    subtype: string;
+    timeout: string;
+    state: string;
+    errname: string;
+    wait: string;
+    id: string;
+}
+interface XmlActionWaitNavigationCfg {
+    type: string;
+    subtype: string;
+    timeout: string;
+    waituntil: string;
+    url: string;
+    errname: string;
+    wait: string;
+    id: string;
+}
+interface XmlActionWaitSleepCfg {
+    type: string;
+    subtype: string;
+    minms: string;
+    maxms: string;
+    errname: string;
+    wait: string;
+    id: string;
+}
+interface XmlFontsvgCfg {
+    exloc: string;
+    inloc: string;
+    csmaptype: string;
+    bsfilter: string;
+}
+interface XmlFontselectorCfg {
+    name: string;
+    fontfamily: string;
+}
+interface XmlFontfamilyCfg {
+    name: string;
+    fontcodes: string;
+    fontchars: string;
+}
+interface XmlFontcodesCfg {
+    name: string;
+    codes: string;
+}
+interface XmlFontcharsCfg {
+    name: string;
+    chars: string;
+}
+type XmlActionConfig = XmlActionBreakCfg | XmlActionCaptchaCfg | XmlActionClickCfg | XmlActionContinueCfg | XmlActionMiscCfg | XmlActionExitCfg | XmlActionExtractCfg | XmlActionGotoCfg | XmlActionHoverCfg | XmlActionIfelseCfg | XmlActionInputCfg | XmlActionInterceptClearCfg | XmlActionInterceptSetCfg | XmlActionLoopdowhileElementCfg | XmlActionLoopdowhileTemplstrCfg | XmlActionLoopforCfg | XmlActionLoopinelesCfg | XmlActionLoopinstrCfg | XmlActionScrollByCfg | XmlActionScrollIntoviewCfg | XmlActionScrollToCfg | XmlActionSelectCfg | XmlActionSetvarDbqueryCfg | XmlActionSetvarElementCfg | XmlActionSetvarFileCfg | XmlActionSetvarGetCfg | XmlActionSetvarOcrCfg | XmlActionSetvarSubtaskCfg | XmlActionSetvarTemplstrCfg | XmlActionSubtaskCfg | XmlActionWaitElementCfg | XmlActionWaitNavigationCfg | XmlActionWaitSleepCfg;
+
+type TokenCaptchaType = "amazon" | "funcaptcha" | "geetest" | "keycaptcha" | "mtcaptcha" | "recaptcha" | "turnstile";
+type RecognitionCaptchaType = "text" | "coordinate" | "grid" | "slider" | "rotation";
+type CaptchaType = TokenCaptchaType | RecognitionCaptchaType;
+
 type TemplateId = number;
+type DomainId = number;
 type HttpHeaders = Record<string, string>;
 interface ScraperStateData extends BrowserStateData {
     /**
@@ -14,6 +419,58 @@ interface ScraperStateData extends BrowserStateData {
      */
     userData: Record<string, string>;
 }
+type InParas = Record<string, string>;
+interface FontttfConfig {
+    exloc: string;
+    inloc: string;
+    minuc: number;
+    maxuc: number;
+    startidx: number;
+    fsfilter: string;
+    fufilter: string;
+    parsetype: string;
+}
+/**
+ * fonts config in xml
+ * * length: 0 ~ 20, default []
+ */
+interface FontsConfig {
+    fontselectorConfig: Record<string, XmlFontselectorCfg>;
+    fontfamilyConfig: Record<string, XmlFontfamilyCfg>;
+    fontcodesConfig: Record<string, XmlFontcodesCfg>;
+    fontcharsConfig: Record<string, XmlFontcharsCfg>;
+    fontsvgCfg?: XmlFontsvgCfg;
+    fontttfConfig?: FontttfConfig;
+}
+type ElementSource = "browser" | "cheerio";
+interface TemplateInScraper {
+    templateId: TemplateId;
+    domainId: DomainId;
+    /**
+     * @default "browser"
+     */
+    defaultElementSource: ElementSource;
+    /**
+     * @default 600 seconds
+     */
+    maxExecutionDuration: number;
+    configDetail: string;
+    capName?: string;
+}
+type AttrsInXml = Record<string, string>;
+type DatatableColumnMap = Map<string, string>;
+interface ParsedTemplate {
+    actionConfigs: XmlActionConfig[];
+    paraCfgs: XmlParaCfg[];
+    fontsConfig: FontsConfig | null;
+    attrsInXml: AttrsInXml;
+    captchaTypes: CaptchaType[];
+    lastUsedTime: number;
+    lastCheckTime: number;
+    datatableMap: Map<string, DatatableColumnMap> | null;
+    template?: TemplateInScraper;
+}
+type ParsedTemplateExt = Required<ParsedTemplate>;
 /**
  * Network context used to execute the task
  */
@@ -184,7 +641,7 @@ interface ScraperConfig {
          */
         clientKey: string;
     };
-    templateUrl?: string;
+    urlPrefix?: string;
     /**
      * the default maximum number of concurrent tasks that can execute the same template in a browserContext
      * @default 1
@@ -231,9 +688,54 @@ interface ScraperConfig {
 
 declare function setScraperLogFun(logFun: LogFunction): boolean;
 
+/** 修改node_modules/xml2js/lib/parser.js文件，添加如下内容（根据tagName自动添加type和subtype属性，如action_setvar_element添加 type="setvar" subtype="element"）：
+          ////////    start of LSD added by Joe  ////////////////////////////////////////////////////////////////////////////////////////////
+          // to be able to add attributes here and later(defaultElementCfg.js), set obj[attrkey] if undefined
+          if(!obj[attrkey]){
+            obj[attrkey] = {}
+          }
+          const subTags = node.name.split("_")
+          if(subTags.length > 1 && typeof obj[attrkey]["type"] ==="undefined"){
+            obj[attrkey]["type"] = subTags[1]
+          }
+          if(subTags.length > 2 && typeof obj[attrkey]["subtype"] ==="undefined"){
+            obj[attrkey]["subtype"] = subTags[2]
+          }
+          ////////    end   of LSD added by Joe  ////////////////////////////////////////////////////////////////////////////////////////////
+ 
+          obj["#name"] = _this.options.tagNameProcessors ? processItem(_this.options.tagNameProcessors, node.name) : node.name;
+ */
+/**
+ * TaskParser
+ */
+declare class TaskParser {
+    #private;
+    /**
+     *
+     * @param xmlStr
+     * @param defaultCfgFlag
+     * @returns {$$:{id, version}, children: {paras: [...], depends: [...], actions: [...]}}}
+     */
+    static convertXmlToJson(xmlStr: string, defaultCfgFlag?: boolean): Promise<any>;
+    static getPartOfJsonCfg(jsonCfg: any, partName: string, optional?: boolean): any;
+    static getParaCfgsFromJsonCfg(jsonCfg: any): XmlParaCfg[];
+    static getAttrsInXml(jsonCfg: any): AttrsInXml;
+    static getCaptchTypes(jsonCfg: any): CaptchaType[];
+    static getDatableMapFromJsonCfg(jsonCfg: any): Map<string, DatatableColumnMap> | null;
+    static getInParas(parasStr: string, paraCfgs?: XmlParaCfg[], splitStr?: string): InParas;
+    static convertExecData(origExecData: ExecData, datatableMap: Map<string, DatatableColumnMap> | null): ExecData;
+}
+
+declare class TemplateManagerInScraper {
+    #private;
+    static parseXmlTemplate(xmlStr: string, datatableFlag?: boolean): Promise<ParsedTemplate>;
+    static getTemplateConfig(templateId: number, xmlStr?: string): Promise<ParsedTemplateExt>;
+    static clearTemplateConfig(templateId?: number): boolean;
+}
+
 declare function performOneTask(templateId: number, parasStr: string, taskNetworkContext: TaskNetworkContext, taskType?: TaskType, xmlStr?: string, taskId?: number, useNickName?: boolean): Promise<TaskResult>;
 
 declare function updateScraperConfig(config: ScraperConfig): Promise<boolean>;
 declare function scraper(newTasks?: TemplateTasks[], config?: ScraperConfig): Promise<boolean>;
 
-export { type BrowserConfig, type ExecData, type ScraperConfig, type TemplatePara, type TemplateTasks, performOneTask, scraper, setScraperLogFun, updateScraperConfig };
+export { type AttrsInXml, type BrowserConfig, type ExecData, type ParsedTemplate, type ScraperConfig, TaskParser, TemplateManagerInScraper, type TemplatePara, type TemplateTasks, performOneTask, scraper, setScraperLogFun, updateScraperConfig };